fix(ui): forward playground custom headers and note billable calls in prompt caching test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
tin 2026-09-17 21:25:01 +00:00
parent 4dedf70434
commit 2da1614bcd
3 changed files with 40 additions and 16 deletions

View file

@ -4,11 +4,13 @@ import React, { useState } from "react";
import { CircleAlert, CircleCheck, CirclePlay } from "lucide-react";
import ModelSelector from "@/components/common_components/ModelSelector";
import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers";
import { getProxyBaseUrl } from "@/components/networking";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardTitle } from "@/components/ui/card";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { getSecureItem } from "@/utils/secureStorage";
import { PromptCachingCallResult, PromptCachingVerdict, runPromptCachingTest } from "./promptCachingTest";
interface PromptCachingTestCardProps {
@ -76,7 +78,13 @@ const PromptCachingTestCard: React.FC<PromptCachingTestCardProps> = ({
setOutcome(null);
setError(null);
try {
const testOptions = { accessToken, model, baseUrl: getProxyBaseUrl(), onCallStart: setRunningCall };
const testOptions = {
accessToken,
model,
baseUrl: getProxyBaseUrl(),
onCallStart: setRunningCall,
customHeaders: customHeadersFromPairs(parseStoredHeaderPairs(getSecureItem("customHeaders"))),
};
setOutcome(await runTestImpl(testOptions));
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
@ -93,7 +101,8 @@ const PromptCachingTestCard: React.FC<PromptCachingTestCardProps> = ({
<CardTitle>Test prompt caching</CardTitle>
<p className="mt-1 break-words text-xs text-muted-foreground">
Sends two identical requests with a ~9k token system prompt and no cache_control of its own. If LiteLLM
injects the breakpoints, the first call writes the cache and the second reads it.
injects the breakpoints, the first call writes the cache and the second reads it. Both calls hit the real
model through the proxy and are billed normally.
</p>
{!enabled && (

View file

@ -89,6 +89,7 @@ describe("runPromptCachingTest", () => {
model: "claude-haiku-4-5",
baseUrl: "http://proxy:4000",
fetchImpl,
customHeaders: { "x-custom-auth": "abc", Authorization: "Bearer wrong" },
};
const result = await runPromptCachingTest(testOptions);
@ -96,6 +97,7 @@ describe("runPromptCachingTest", () => {
const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit];
expect(url).toBe("http://proxy:4000/v1/chat/completions");
expect((init.headers as Record<string, string>).Authorization).toBe("Bearer token-123");
expect((init.headers as Record<string, string>)["x-custom-auth"]).toBe("abc");
expect((init.headers as Record<string, string>)["x-litellm-tags"]).toBe("prompt-caching-test");
const serialized = JSON.stringify(seenBodies);

View file

@ -1,6 +1,6 @@
import { z } from "zod";
import { buildPlaygroundHeaders } from "@/components/llm_calls/request_headers";
import { CustomHeaders, buildPlaygroundHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers";
import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096;
@ -93,20 +93,29 @@ const parseResponseCost = (headers: Headers): number | null => {
return Number.isFinite(cost) ? cost : null;
};
const sendTestCall = async (
accessToken: string,
baseUrl: string,
body: PromptCachingTestRequestBody,
fetchImpl: typeof fetch,
): Promise<PromptCachingCallResult> => {
interface SendTestCallOptions {
readonly accessToken: string;
readonly baseUrl: string;
readonly body: PromptCachingTestRequestBody;
readonly fetchImpl: typeof fetch;
readonly customHeaders: CustomHeaders;
}
const sendTestCall = async ({
accessToken,
baseUrl,
body,
fetchImpl,
customHeaders,
}: SendTestCallOptions): Promise<PromptCachingCallResult> => {
const startedAt = Date.now();
const headers = withRequiredHeaders(buildPlaygroundHeaders(["prompt-caching-test"], customHeaders), {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
});
const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, {
method: "POST",
headers: {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
...buildPlaygroundHeaders(["prompt-caching-test"]),
},
headers,
body: JSON.stringify(body),
});
const durationMs = Date.now() - startedAt;
@ -137,6 +146,7 @@ export interface RunPromptCachingTestOptions {
readonly baseUrl: string;
readonly fetchImpl?: typeof fetch;
readonly onCallStart?: (callIndex: 1 | 2) => void;
readonly customHeaders?: CustomHeaders;
}
export const runPromptCachingTest = async ({
@ -145,6 +155,7 @@ export const runPromptCachingTest = async ({
baseUrl,
fetchImpl = fetch,
onCallStart,
customHeaders = {},
}: RunPromptCachingTestOptions): Promise<{
first: PromptCachingCallResult;
second: PromptCachingCallResult;
@ -153,10 +164,12 @@ export const runPromptCachingTest = async ({
const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`);
const body = buildTestRequestBody(model, systemPrompt);
const callOptions = { accessToken, baseUrl, body, fetchImpl, customHeaders };
onCallStart?.(1);
const first = await sendTestCall(accessToken, baseUrl, body, fetchImpl);
const first = await sendTestCall(callOptions);
onCallStart?.(2);
const second = await sendTestCall(accessToken, baseUrl, body, fetchImpl);
const second = await sendTestCall(callOptions);
return { first, second, verdict: judgePromptCachingTest(first, second) };
};