diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx index 4ee13000f6f..defe64cd422 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTestCard.tsx @@ -4,11 +4,13 @@ import React, { useState } from "react"; import { CircleAlert, CircleCheck, CirclePlay } from "lucide-react"; import ModelSelector from "@/components/common_components/ModelSelector"; +import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers"; import { getProxyBaseUrl } from "@/components/networking"; import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { getSecureItem } from "@/utils/secureStorage"; import { PromptCachingCallResult, PromptCachingVerdict, runPromptCachingTest } from "./promptCachingTest"; interface PromptCachingTestCardProps { @@ -76,7 +78,13 @@ const PromptCachingTestCard: React.FC = ({ setOutcome(null); setError(null); try { - const testOptions = { accessToken, model, baseUrl: getProxyBaseUrl(), onCallStart: setRunningCall }; + const testOptions = { + accessToken, + model, + baseUrl: getProxyBaseUrl(), + onCallStart: setRunningCall, + customHeaders: customHeadersFromPairs(parseStoredHeaderPairs(getSecureItem("customHeaders"))), + }; setOutcome(await runTestImpl(testOptions)); } catch (e) { setError(e instanceof Error ? e.message : String(e)); @@ -93,7 +101,8 @@ const PromptCachingTestCard: React.FC = ({ Test prompt caching

Sends two identical requests with a ~9k token system prompt and no cache_control of its own. If LiteLLM - injects the breakpoints, the first call writes the cache and the second reads it. + injects the breakpoints, the first call writes the cache and the second reads it. Both calls hit the real + model through the proxy and are billed normally.

{!enabled && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts index f638a19862d..b16f5f30be6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.test.ts @@ -89,6 +89,7 @@ describe("runPromptCachingTest", () => { model: "claude-haiku-4-5", baseUrl: "http://proxy:4000", fetchImpl, + customHeaders: { "x-custom-auth": "abc", Authorization: "Bearer wrong" }, }; const result = await runPromptCachingTest(testOptions); @@ -96,6 +97,7 @@ describe("runPromptCachingTest", () => { const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]; expect(url).toBe("http://proxy:4000/v1/chat/completions"); expect((init.headers as Record).Authorization).toBe("Bearer token-123"); + expect((init.headers as Record)["x-custom-auth"]).toBe("abc"); expect((init.headers as Record)["x-litellm-tags"]).toBe("prompt-caching-test"); const serialized = JSON.stringify(seenBodies); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts index 302b685a895..80f940bf9dd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/promptCachingTest.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { buildPlaygroundHeaders } from "@/components/llm_calls/request_headers"; +import { CustomHeaders, buildPlaygroundHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096; @@ -93,20 +93,29 @@ const parseResponseCost = (headers: Headers): number | null => { return Number.isFinite(cost) ? cost : null; }; -const sendTestCall = async ( - accessToken: string, - baseUrl: string, - body: PromptCachingTestRequestBody, - fetchImpl: typeof fetch, -): Promise => { +interface SendTestCallOptions { + readonly accessToken: string; + readonly baseUrl: string; + readonly body: PromptCachingTestRequestBody; + readonly fetchImpl: typeof fetch; + readonly customHeaders: CustomHeaders; +} + +const sendTestCall = async ({ + accessToken, + baseUrl, + body, + fetchImpl, + customHeaders, +}: SendTestCallOptions): Promise => { const startedAt = Date.now(); + const headers = withRequiredHeaders(buildPlaygroundHeaders(["prompt-caching-test"], customHeaders), { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }); const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, { method: "POST", - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - ...buildPlaygroundHeaders(["prompt-caching-test"]), - }, + headers, body: JSON.stringify(body), }); const durationMs = Date.now() - startedAt; @@ -137,6 +146,7 @@ export interface RunPromptCachingTestOptions { readonly baseUrl: string; readonly fetchImpl?: typeof fetch; readonly onCallStart?: (callIndex: 1 | 2) => void; + readonly customHeaders?: CustomHeaders; } export const runPromptCachingTest = async ({ @@ -145,6 +155,7 @@ export const runPromptCachingTest = async ({ baseUrl, fetchImpl = fetch, onCallStart, + customHeaders = {}, }: RunPromptCachingTestOptions): Promise<{ first: PromptCachingCallResult; second: PromptCachingCallResult; @@ -153,10 +164,12 @@ export const runPromptCachingTest = async ({ const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`); const body = buildTestRequestBody(model, systemPrompt); + const callOptions = { accessToken, baseUrl, body, fetchImpl, customHeaders }; + onCallStart?.(1); - const first = await sendTestCall(accessToken, baseUrl, body, fetchImpl); + const first = await sendTestCall(callOptions); onCallStart?.(2); - const second = await sendTestCall(accessToken, baseUrl, body, fetchImpl); + const second = await sendTestCall(callOptions); return { first, second, verdict: judgePromptCachingTest(first, second) }; };