From 5bec2426299751737475f07a4735afcaf1d00c90 Mon Sep 17 00:00:00 2001 From: tin Date: Thu, 17 Sep 2026 20:41:42 +0000 Subject: [PATCH 1/5] feat(ui): prompt caching test button on the cost optimization tab Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/PromptCachingTab.test.tsx | 79 +++++++-- .../_components/PromptCachingTab.tsx | 6 + .../PromptCachingTestCard.test.tsx | 68 ++++++++ .../_components/PromptCachingTestCard.tsx | 164 ++++++++++++++++++ .../_components/promptCachingTest.test.ts | 124 +++++++++++++ .../_components/promptCachingTest.ts | 162 +++++++++++++++++ .../_components/general_settings.tsx | 4 +- 7 files changed, 591 insertions(+), 16 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx index 66db347e70f..9bba3d5495a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx @@ -7,11 +7,17 @@ vi.mock("@/components/networking", () => ({ getGeneralSettingsCall: (...args: unknown[]) => mockGetGeneralSettingsCall(...args), })); -vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({ - PromptCachingPanel: () =>
, -})); +vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => { + const original = + await importOriginal(); + return { + ...original, + PromptCachingPanel: () =>
, + }; +}); const mockCacheLeakageCard = vi.fn(); +const mockPromptCachingTestCard = vi.fn(); vi.mock("./CacheLeakageCard", () => ({ __esModule: true, @@ -21,27 +27,72 @@ vi.mock("./CacheLeakageCard", () => ({ }, })); +vi.mock("./PromptCachingTestCard", () => ({ + __esModule: true, + default: (props: unknown) => { + mockPromptCachingTestCard(props); + return
; + }, +})); + import PromptCachingTab from "./PromptCachingTab"; +const activity = { + dateValue: {}, + onDateChange: vi.fn(), + results: [], + loading: false, + isFetchingMore: false, + progress: { currentPage: 1, totalPages: 1 }, + cancelled: false, + failed: false, + cancel: vi.fn(), +}; + describe("PromptCachingTab", () => { it("renders the cache leakage table alongside the caching settings", async () => { mockGetGeneralSettingsCall.mockResolvedValue([]); - const activity = { - dateValue: {}, - onDateChange: vi.fn(), - results: [], - loading: false, - isFetchingMore: false, - progress: { currentPage: 1, totalPages: 1 }, - cancelled: false, - failed: false, - cancel: vi.fn(), - }; render(); expect(screen.getByTestId("caching-settings")).toBeInTheDocument(); expect(screen.getByTestId("cache-leakage-card")).toBeInTheDocument(); await waitFor(() => expect(mockCacheLeakageCard).toHaveBeenCalledWith(expect.objectContaining({ activity }))); }); + + it("passes enabled=true to the test card when the caching setting is on", async () => { + mockGetGeneralSettingsCall.mockResolvedValue([ + { + field_name: "enable_anthropic_prompt_caching", + field_type: "Boolean", + field_value: true, + field_description: "", + stored_in_db: true, + }, + ]); + + render(); + + await waitFor(() => + expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: true })), + ); + }); + + it("passes enabled=false to the test card when the caching setting is off", async () => { + mockGetGeneralSettingsCall.mockResolvedValue([ + { + field_name: "enable_anthropic_prompt_caching", + field_type: "Boolean", + field_value: false, + field_description: "", + stored_in_db: true, + }, + ]); + + render(); + + await waitFor(() => + expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: false })), + ); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx index 59b38f272e0..dd541d7563b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx @@ -5,10 +5,13 @@ import React, { useCallback, useEffect, useState } from "react"; import { getGeneralSettingsCall } from "@/components/networking"; import { toast } from "@/lib/toast"; import { + ENABLE_ANTHROPIC_PROMPT_CACHING, PromptCachingPanel, generalSettingsItem, + isOn, } from "@/app/(dashboard)/router-settings/_components/general_settings"; import CacheLeakageCard from "./CacheLeakageCard"; +import PromptCachingTestCard from "./PromptCachingTestCard"; import { DailyActivityRange } from "./useDailyActivityRange"; interface PromptCachingTabProps { @@ -45,9 +48,12 @@ const PromptCachingTab: React.FC = ({ accessToken, activi return null; } + const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING); + return (
+
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx new file mode 100644 index 00000000000..b8dab16cf24 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx @@ -0,0 +1,68 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/components/networking", () => ({ + getProxyBaseUrl: vi.fn(() => "http://proxy:4000"), +})); + +vi.mock("@/components/common_components/ModelSelector", () => ({ + __esModule: true, + default: ({ onChange }: { onChange?: (value: string | null) => void }) => ( + + {running && Sending call {runningCall} of 2...} +
+ + {error && ( + + + Test failed + {error} + + )} + + {outcome && ( +
+ + + + + Prompt tokens + Cache write + Cache read + Cost + Latency + + + + + + +
+ {verdict && ( + + {outcome.verdict === "not_injected" ? : } + {verdict.text(outcome)} + + )} +
+ )} + + + ); +}; + +export default PromptCachingTestCard; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts new file mode 100644 index 00000000000..76cc5ca88bd --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS, + PromptCachingCallResult, + buildTestSystemPrompt, + judgePromptCachingTest, + runPromptCachingTest, +} from "./promptCachingTest"; + +const callResult = (overrides: Partial = {}): PromptCachingCallResult => ({ + cacheCreationTokens: 0, + cacheReadTokens: 0, + promptTokens: 5000, + responseCost: null, + model: null, + durationMs: 10, + ...overrides, +}); + +const jsonResponse = (body: unknown, init: { status?: number; headers?: Record } = {}) => + new Response(JSON.stringify(body), { + status: init.status ?? 200, + headers: { "Content-Type": "application/json", ...init.headers }, + }); + +const anthropicUsage = { + prompt_tokens: 5010, + cache_creation_input_tokens: 5000, + cache_read_input_tokens: 0, +}; + +describe("buildTestSystemPrompt", () => { + it("is comfortably above Anthropic's largest minimum cacheable size and differs by nonce", () => { + const first = buildTestSystemPrompt("run-a"); + const second = buildTestSystemPrompt("run-b"); + + expect(first.length).toBeGreaterThan(20000); + expect(first.length).toBeGreaterThan(PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS * 4); + expect(first).not.toBe(second); + expect(first.startsWith("Test run run-a.")).toBe(true); + expect(first.slice(first.indexOf("\n"))).toBe(second.slice(second.indexOf("\n"))); + }); +}); + +describe("judgePromptCachingTest", () => { + it("returns injected when call 1 writes and call 2 reads", () => { + expect( + judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult({ cacheReadTokens: 5000 })), + ).toBe("injected"); + }); + + it("returns cache_hit_only when neither call writes but a call reads", () => { + expect(judgePromptCachingTest(callResult(), callResult({ cacheReadTokens: 5000 }))).toBe("cache_hit_only"); + expect(judgePromptCachingTest(callResult({ cacheReadTokens: 5000 }), callResult({ cacheReadTokens: 5000 }))).toBe( + "cache_hit_only", + ); + }); + + it("returns not_injected when call 1 writes but call 2 reads nothing", () => { + expect(judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult())).toBe("not_injected"); + }); + + it("returns not_injected with no cache activity at all", () => { + expect(judgePromptCachingTest(callResult(), callResult())).toBe("not_injected"); + }); +}); + +describe("runPromptCachingTest", () => { + it("sends two sequential identical requests with no cache_control and parses usage", async () => { + const seenBodies: unknown[] = []; + const responses = [ + jsonResponse( + { model: "claude-haiku-4-5", usage: anthropicUsage }, + { headers: { "x-litellm-response-cost": "0.0042" } }, + ), + jsonResponse({ + model: "claude-haiku-4-5", + usage: { prompt_tokens: 5010, prompt_tokens_details: { cached_tokens: 5000 } }, + }), + ]; + const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => { + seenBodies.push(JSON.parse(String(init?.body))); + return responses[seenBodies.length - 1]; + }); + + const result = await runPromptCachingTest({ + accessToken: "token-123", + model: "claude-haiku-4-5", + baseUrl: "http://proxy:4000", + fetchImpl, + }); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe("http://proxy:4000/v1/chat/completions"); + expect((init.headers as Record).Authorization).toBe("Bearer token-123"); + expect((init.headers as Record)["x-litellm-tags"]).toBe("prompt-caching-test"); + + const serialized = JSON.stringify(seenBodies); + expect(serialized).not.toContain("cache_control"); + expect((seenBodies[0] as { messages: { content: string }[] }).messages[0].content).toBe( + (seenBodies[1] as { messages: { content: string }[] }).messages[0].content, + ); + + expect(result.first.cacheCreationTokens).toBe(5000); + expect(result.first.responseCost).toBeCloseTo(0.0042); + expect(result.second.cacheReadTokens).toBe(5000); + expect(result.verdict).toBe("injected"); + }); + + it("rejects with status and error message on non-2xx", async () => { + const fetchImpl = vi.fn(async () => jsonResponse({ error: { message: "boom" } }, { status: 400 })); + + await expect( + runPromptCachingTest({ + accessToken: "token-123", + model: "claude-haiku-4-5", + baseUrl: "http://proxy:4000", + fetchImpl, + }), + ).rejects.toThrow(/400.*boom|boom.*400/); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts new file mode 100644 index 00000000000..60ada4061f4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts @@ -0,0 +1,162 @@ +import { z } from "zod"; + +import { buildPlaygroundHeaders } from "@/components/llm_calls/request_headers"; +import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; + +export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096; + +const FILLER_LINE_COUNT = 600; + +export const buildTestSystemPrompt = (nonce: string): string => { + const filler = Array.from( + { length: FILLER_LINE_COUNT }, + (_, index) => `Reference item ${index + 1}: the gateway forwards requests to the configured deployment.`, + ).join("\n"); + return `Test run ${nonce}.\n${filler}`; +}; + +export interface PromptCachingTestRequestBody { + readonly model: string; + readonly messages: readonly { readonly role: string; readonly content: string }[]; + readonly max_tokens: number; + readonly temperature: number; +} + +export const buildTestRequestBody = (model: string, systemPrompt: string): PromptCachingTestRequestBody => ({ + model, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: "Reply with the single word OK." }, + ], + max_tokens: 5, + temperature: 0, +}); + +export interface PromptCachingCallResult { + cacheCreationTokens: number; + cacheReadTokens: number; + promptTokens: number; + responseCost: number | null; + model: string | null; + durationMs: number; +} + +export type PromptCachingVerdict = "injected" | "not_injected" | "cache_hit_only"; + +export const judgePromptCachingTest = ( + first: PromptCachingCallResult, + second: PromptCachingCallResult, +): PromptCachingVerdict => { + if (first.cacheCreationTokens > 0 && second.cacheReadTokens > 0) { + return "injected"; + } + if (first.cacheCreationTokens === 0 && second.cacheCreationTokens === 0) { + if (first.cacheReadTokens > 0 || second.cacheReadTokens > 0) { + return "cache_hit_only"; + } + } + return "not_injected"; +}; + +const tokenDetailsSchema = z + .object({ + cached_tokens: z.number().nullish(), + cache_write_tokens: z.number().nullish(), + }) + .nullish(); + +const usageSchema = z + .object({ + prompt_tokens: z.number().nullish(), + cache_creation_input_tokens: z.number().nullish(), + cache_read_input_tokens: z.number().nullish(), + prompt_tokens_details: tokenDetailsSchema, + input_tokens_details: tokenDetailsSchema, + }) + .nullish(); + +const chatCompletionResponseSchema = z.object({ + model: z.string().nullish(), + usage: usageSchema, +}); + +const errorBodySchema = z.object({ + error: z.object({ message: z.string().optional() }).optional(), +}); + +const parseResponseCost = (headers: Headers): number | null => { + const raw = headers.get("x-litellm-response-cost"); + if (raw === null) { + return null; + } + const cost = Number.parseFloat(raw); + return Number.isFinite(cost) ? cost : null; +}; + +const sendTestCall = async ( + accessToken: string, + baseUrl: string, + body: PromptCachingTestRequestBody, + fetchImpl: typeof fetch, +): Promise => { + const startedAt = Date.now(); + const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + ...buildPlaygroundHeaders(["prompt-caching-test"]), + }, + body: JSON.stringify(body), + }); + const durationMs = Date.now() - startedAt; + + const json: unknown = await response.json(); + if (!response.ok) { + const parsedError = errorBodySchema.safeParse(json); + const detail = parsedError.success ? parsedError.data.error?.message : undefined; + throw new Error(`Request failed with status ${response.status}${detail ? `: ${detail}` : ""}`); + } + + const parsed = chatCompletionResponseSchema.parse(json); + const cacheTokens = extractPromptCacheTokens(parsed.usage); + + return { + cacheCreationTokens: cacheTokens.cacheCreationTokens ?? 0, + cacheReadTokens: cacheTokens.cacheReadTokens ?? 0, + promptTokens: parsed.usage?.prompt_tokens ?? 0, + responseCost: parseResponseCost(response.headers), + model: parsed.model ?? null, + durationMs, + }; +}; + +export interface RunPromptCachingTestOptions { + readonly accessToken: string; + readonly model: string; + readonly baseUrl: string; + readonly fetchImpl?: typeof fetch; + readonly onCallStart?: (callIndex: 1 | 2) => void; +} + +export const runPromptCachingTest = async ({ + accessToken, + model, + baseUrl, + fetchImpl = fetch, + onCallStart, +}: RunPromptCachingTestOptions): Promise<{ + first: PromptCachingCallResult; + second: PromptCachingCallResult; + verdict: PromptCachingVerdict; +}> => { + const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`); + const body = buildTestRequestBody(model, systemPrompt); + + onCallStart?.(1); + const first = await sendTestCall(accessToken, baseUrl, body, fetchImpl); + onCallStart?.(2); + const second = await sendTestCall(accessToken, baseUrl, body, fetchImpl); + + return { first, second, verdict: judgePromptCachingTest(first, second) }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 9a718cbe9b8..9ccbc7a3391 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -16,11 +16,11 @@ import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks" import RoutingGroups from "@/components/routing_groups"; const PROMPT_CACHING_TAB = "prompt_caching"; -const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching"; +export const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching"; const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl"; const OPENAI_SYSTEM_MESSAGES_FIRST = "openai_system_messages_first"; -const isOn = (value: unknown) => value === true || value === "true"; +export const isOn = (value: unknown) => value === true || value === "true"; interface GeneralSettingsPageProps { accessToken: string | null; From 865424fb1ced2be73f079993a54b6afa357dc5ff Mon Sep 17 00:00:00 2001 From: tin Date: Thu, 17 Sep 2026 20:54:01 +0000 Subject: [PATCH 2/5] fix(ui): expose response cost header and correct prompt caching test copy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + tests/test_litellm/proxy/test_proxy_server.py | 7 +++++++ .../_components/PromptCachingTestCard.tsx | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index 8409a161800..c44d2a13c7f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -205,6 +205,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [ "x-litellm-guardrail-scan-id", "x-litellm-guardrail-scan-metadata", "x-litellm-cache-key", + "x-litellm-response-cost", ] # Gemini model-specific minimal thinking budget constants diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 41c4956dba6..e973e154c61 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -102,6 +102,13 @@ def test_cors_exposes_cache_key_header_to_browser_js(): assert "x-litellm-cache-key" in cors_middleware.kwargs["expose_headers"] +def test_cors_exposes_response_cost_header_to_browser_js(): + from fastapi.middleware.cors import CORSMiddleware + + cors_middleware = next(m for m in app.user_middleware if m.cls is CORSMiddleware) + assert "x-litellm-response-cost" in cors_middleware.kwargs["expose_headers"] + + def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): mock_login_result = {"user_id": "test-user"} mock_prisma_client = MagicMock() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx index 06b58a51da5..e406224408a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx @@ -91,7 +91,7 @@ const PromptCachingTestCard: React.FC = ({ Test prompt caching

- Sends two identical requests with a ~5k token system prompt and no cache_control of its own. If LiteLLM + Sends two identical requests with a ~9k token system prompt and no cache_control of its own. If LiteLLM injects the breakpoints, the first call writes the cache and the second reads it.

From 4dedf70434ad6a16da4b504542996c3dfb50f501 Mon Sep 17 00:00:00 2001 From: tin Date: Thu, 17 Sep 2026 21:12:05 +0000 Subject: [PATCH 3/5] fix(ui): fix stale settings mock and inline object lint warnings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../CostOptimizationView.activity.test.tsx | 11 +++++++--- .../_components/PromptCachingTestCard.tsx | 3 ++- .../_components/promptCachingTest.test.ts | 20 +++++++++---------- .../_components/promptCachingTest.ts | 18 ++++++++--------- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx index 03250e3e53b..b3ba951783f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -37,9 +37,14 @@ vi.mock("@/components/shared/charts", () => ({ SEQUENTIAL_COLOR_RAMP: ["indigo"], })); -vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({ - PromptCachingPanel: () =>
, -})); +vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => { + const original = + await importOriginal(); + return { + ...original, + PromptCachingPanel: () =>
, + }; +}); vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx index e406224408a..4ee13000f6f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx @@ -76,7 +76,8 @@ const PromptCachingTestCard: React.FC = ({ setOutcome(null); setError(null); try { - setOutcome(await runTestImpl({ accessToken, model, baseUrl: getProxyBaseUrl(), onCallStart: setRunningCall })); + const testOptions = { accessToken, model, baseUrl: getProxyBaseUrl(), onCallStart: setRunningCall }; + setOutcome(await runTestImpl(testOptions)); } catch (e) { setError(e instanceof Error ? e.message : String(e)); } finally { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts index 76cc5ca88bd..f638a19862d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts @@ -84,12 +84,13 @@ describe("runPromptCachingTest", () => { return responses[seenBodies.length - 1]; }); - const result = await runPromptCachingTest({ + const testOptions = { accessToken: "token-123", model: "claude-haiku-4-5", baseUrl: "http://proxy:4000", fetchImpl, - }); + }; + const result = await runPromptCachingTest(testOptions); expect(fetchImpl).toHaveBeenCalledTimes(2); const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]; @@ -112,13 +113,12 @@ describe("runPromptCachingTest", () => { it("rejects with status and error message on non-2xx", async () => { const fetchImpl = vi.fn(async () => jsonResponse({ error: { message: "boom" } }, { status: 400 })); - await expect( - runPromptCachingTest({ - accessToken: "token-123", - model: "claude-haiku-4-5", - baseUrl: "http://proxy:4000", - fetchImpl, - }), - ).rejects.toThrow(/400.*boom|boom.*400/); + const testOptions = { + accessToken: "token-123", + model: "claude-haiku-4-5", + baseUrl: "http://proxy:4000", + fetchImpl, + }; + await expect(runPromptCachingTest(testOptions)).rejects.toThrow(/400.*boom|boom.*400/); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts index 60ada4061f4..302b685a895 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts @@ -65,15 +65,15 @@ const tokenDetailsSchema = z }) .nullish(); -const usageSchema = z - .object({ - prompt_tokens: z.number().nullish(), - cache_creation_input_tokens: z.number().nullish(), - cache_read_input_tokens: z.number().nullish(), - prompt_tokens_details: tokenDetailsSchema, - input_tokens_details: tokenDetailsSchema, - }) - .nullish(); +const usageFields = { + prompt_tokens: z.number().nullish(), + cache_creation_input_tokens: z.number().nullish(), + cache_read_input_tokens: z.number().nullish(), + prompt_tokens_details: tokenDetailsSchema, + input_tokens_details: tokenDetailsSchema, +}; + +const usageSchema = z.object(usageFields).nullish(); const chatCompletionResponseSchema = z.object({ model: z.string().nullish(), From 2da1614bcd66240597759f9aa2c9558ecd8e022f Mon Sep 17 00:00:00 2001 From: tin Date: Thu, 17 Sep 2026 21:25:01 +0000 Subject: [PATCH 4/5] fix(ui): forward playground custom headers and note billable calls in prompt caching test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/PromptCachingTestCard.tsx | 13 +++++- .../_components/promptCachingTest.test.ts | 2 + .../_components/promptCachingTest.ts | 41 ++++++++++++------- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx index 4ee13000f6f..defe64cd422 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx @@ -4,11 +4,13 @@ import React, { useState } from "react"; import { CircleAlert, CircleCheck, CirclePlay } from "lucide-react"; import ModelSelector from "@/components/common_components/ModelSelector"; +import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers"; import { getProxyBaseUrl } from "@/components/networking"; import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { getSecureItem } from "@/utils/secureStorage"; import { PromptCachingCallResult, PromptCachingVerdict, runPromptCachingTest } from "./promptCachingTest"; interface PromptCachingTestCardProps { @@ -76,7 +78,13 @@ const PromptCachingTestCard: React.FC = ({ setOutcome(null); setError(null); try { - const testOptions = { accessToken, model, baseUrl: getProxyBaseUrl(), onCallStart: setRunningCall }; + const testOptions = { + accessToken, + model, + baseUrl: getProxyBaseUrl(), + onCallStart: setRunningCall, + customHeaders: customHeadersFromPairs(parseStoredHeaderPairs(getSecureItem("customHeaders"))), + }; setOutcome(await runTestImpl(testOptions)); } catch (e) { setError(e instanceof Error ? e.message : String(e)); @@ -93,7 +101,8 @@ const PromptCachingTestCard: React.FC = ({ Test prompt caching

Sends two identical requests with a ~9k token system prompt and no cache_control of its own. If LiteLLM - injects the breakpoints, the first call writes the cache and the second reads it. + injects the breakpoints, the first call writes the cache and the second reads it. Both calls hit the real + model through the proxy and are billed normally.

{!enabled && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts index f638a19862d..b16f5f30be6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts @@ -89,6 +89,7 @@ describe("runPromptCachingTest", () => { model: "claude-haiku-4-5", baseUrl: "http://proxy:4000", fetchImpl, + customHeaders: { "x-custom-auth": "abc", Authorization: "Bearer wrong" }, }; const result = await runPromptCachingTest(testOptions); @@ -96,6 +97,7 @@ describe("runPromptCachingTest", () => { const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]; expect(url).toBe("http://proxy:4000/v1/chat/completions"); expect((init.headers as Record).Authorization).toBe("Bearer token-123"); + expect((init.headers as Record)["x-custom-auth"]).toBe("abc"); expect((init.headers as Record)["x-litellm-tags"]).toBe("prompt-caching-test"); const serialized = JSON.stringify(seenBodies); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts index 302b685a895..80f940bf9dd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { buildPlaygroundHeaders } from "@/components/llm_calls/request_headers"; +import { CustomHeaders, buildPlaygroundHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096; @@ -93,20 +93,29 @@ const parseResponseCost = (headers: Headers): number | null => { return Number.isFinite(cost) ? cost : null; }; -const sendTestCall = async ( - accessToken: string, - baseUrl: string, - body: PromptCachingTestRequestBody, - fetchImpl: typeof fetch, -): Promise => { +interface SendTestCallOptions { + readonly accessToken: string; + readonly baseUrl: string; + readonly body: PromptCachingTestRequestBody; + readonly fetchImpl: typeof fetch; + readonly customHeaders: CustomHeaders; +} + +const sendTestCall = async ({ + accessToken, + baseUrl, + body, + fetchImpl, + customHeaders, +}: SendTestCallOptions): Promise => { const startedAt = Date.now(); + const headers = withRequiredHeaders(buildPlaygroundHeaders(["prompt-caching-test"], customHeaders), { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }); const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, { method: "POST", - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - ...buildPlaygroundHeaders(["prompt-caching-test"]), - }, + headers, body: JSON.stringify(body), }); const durationMs = Date.now() - startedAt; @@ -137,6 +146,7 @@ export interface RunPromptCachingTestOptions { readonly baseUrl: string; readonly fetchImpl?: typeof fetch; readonly onCallStart?: (callIndex: 1 | 2) => void; + readonly customHeaders?: CustomHeaders; } export const runPromptCachingTest = async ({ @@ -145,6 +155,7 @@ export const runPromptCachingTest = async ({ baseUrl, fetchImpl = fetch, onCallStart, + customHeaders = {}, }: RunPromptCachingTestOptions): Promise<{ first: PromptCachingCallResult; second: PromptCachingCallResult; @@ -153,10 +164,12 @@ export const runPromptCachingTest = async ({ const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`); const body = buildTestRequestBody(model, systemPrompt); + const callOptions = { accessToken, baseUrl, body, fetchImpl, customHeaders }; + onCallStart?.(1); - const first = await sendTestCall(accessToken, baseUrl, body, fetchImpl); + const first = await sendTestCall(callOptions); onCallStart?.(2); - const second = await sendTestCall(accessToken, baseUrl, body, fetchImpl); + const second = await sendTestCall(callOptions); return { first, second, verdict: judgePromptCachingTest(first, second) }; }; From 1e891e486ee89e7798d29e93555af0826d4a5794 Mon Sep 17 00:00:00 2001 From: tin Date: Thu, 17 Sep 2026 21:40:18 +0000 Subject: [PATCH 5/5] fix(ui): distinguish injected cache writes that miss on the read Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/PromptCachingTestCard.test.tsx | 15 +++++++++++++++ .../_components/PromptCachingTestCard.tsx | 15 +++++++++++++-- .../_components/promptCachingTest.test.ts | 4 ++-- .../_components/promptCachingTest.ts | 5 ++++- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx index b8dab16cf24..3bb1045c8c1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.test.tsx @@ -54,6 +54,21 @@ describe("PromptCachingTestCard", () => { ); }); + it("shows the injected_no_read verdict when call 2 misses the cache", async () => { + const runTest = vi.fn(async () => ({ + ...successOutcome, + second: { ...successOutcome.second, cacheReadTokens: 0 }, + verdict: "injected_no_read" as const, + })); + render(); + + fireEvent.click(screen.getByTestId("model-selector")); + fireEvent.click(screen.getByRole("button", { name: /run test/i })); + + expect(await screen.findByText(/cache_control injected, but the second call missed the cache/)).toBeInTheDocument(); + expect(screen.getByText(/Call 1 wrote 5000 tokens to the cache but call 2 read 0/)).toBeInTheDocument(); + }); + it("shows an error alert when the run throws", async () => { const runTest = vi.fn(async () => { throw new Error("Request failed with status 400: boom"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx index defe64cd422..4cc1db49ec1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx @@ -29,13 +29,19 @@ const formatCost = (cost: number | null) => (cost === null ? "-" : `$${cost.toFi const VERDICT_COPY: Record< PromptCachingVerdict, - { variant: "success" | "info" | "error"; text: (o: TestOutcome) => string } + { variant: "success" | "info" | "warning" | "error"; title?: string; text: (o: TestOutcome) => string } > = { injected: { variant: "success", text: (o) => `LiteLLM injected cache_control: call 1 wrote ${o.first.cacheCreationTokens} tokens to the cache and call 2 read ${o.second.cacheReadTokens} tokens.`, }, + injected_no_read: { + variant: "warning", + title: "cache_control injected, but the second call missed the cache", + text: (o) => + `Call 1 wrote ${o.first.cacheCreationTokens} tokens to the cache but call 2 read 0. LiteLLM is injecting the breakpoints; check the cache TTL or whether both calls reached the same upstream credentials.`, + }, cache_hit_only: { variant: "info", text: () => "Cache read on both calls: cache_control was on the wire and a warm cache already existed.", @@ -160,7 +166,12 @@ const PromptCachingTestCard: React.FC = ({ {verdict && ( - {outcome.verdict === "not_injected" ? : } + {outcome.verdict === "injected" || outcome.verdict === "cache_hit_only" ? ( + + ) : ( + + )} + {verdict.title && {verdict.title}} {verdict.text(outcome)} )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts index b16f5f30be6..649dce7cce3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts @@ -57,8 +57,8 @@ describe("judgePromptCachingTest", () => { ); }); - it("returns not_injected when call 1 writes but call 2 reads nothing", () => { - expect(judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult())).toBe("not_injected"); + it("returns injected_no_read when call 1 writes but call 2 reads nothing", () => { + expect(judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult())).toBe("injected_no_read"); }); it("returns not_injected with no cache activity at all", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts index 80f940bf9dd..f0c2102523d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts @@ -41,7 +41,7 @@ export interface PromptCachingCallResult { durationMs: number; } -export type PromptCachingVerdict = "injected" | "not_injected" | "cache_hit_only"; +export type PromptCachingVerdict = "injected" | "injected_no_read" | "not_injected" | "cache_hit_only"; export const judgePromptCachingTest = ( first: PromptCachingCallResult, @@ -50,6 +50,9 @@ export const judgePromptCachingTest = ( if (first.cacheCreationTokens > 0 && second.cacheReadTokens > 0) { return "injected"; } + if (first.cacheCreationTokens > 0 && second.cacheReadTokens === 0) { + return "injected_no_read"; + } if (first.cacheCreationTokens === 0 && second.cacheCreationTokens === 0) { if (first.cacheReadTokens > 0 || second.cacheReadTokens > 0) { return "cache_hit_only";