mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(ui): forward playground custom headers and note billable calls in prompt caching test
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
4dedf70434
commit
2da1614bcd
3 changed files with 40 additions and 16 deletions
|
|
@ -4,11 +4,13 @@ import React, { useState } from "react";
|
|||
import { CircleAlert, CircleCheck, CirclePlay } from "lucide-react";
|
||||
|
||||
import ModelSelector from "@/components/common_components/ModelSelector";
|
||||
import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers";
|
||||
import { getProxyBaseUrl } from "@/components/networking";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardTitle } from "@/components/ui/card";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { getSecureItem } from "@/utils/secureStorage";
|
||||
import { PromptCachingCallResult, PromptCachingVerdict, runPromptCachingTest } from "./promptCachingTest";
|
||||
|
||||
interface PromptCachingTestCardProps {
|
||||
|
|
@ -76,7 +78,13 @@ const PromptCachingTestCard: React.FC<PromptCachingTestCardProps> = ({
|
|||
setOutcome(null);
|
||||
setError(null);
|
||||
try {
|
||||
const testOptions = { accessToken, model, baseUrl: getProxyBaseUrl(), onCallStart: setRunningCall };
|
||||
const testOptions = {
|
||||
accessToken,
|
||||
model,
|
||||
baseUrl: getProxyBaseUrl(),
|
||||
onCallStart: setRunningCall,
|
||||
customHeaders: customHeadersFromPairs(parseStoredHeaderPairs(getSecureItem("customHeaders"))),
|
||||
};
|
||||
setOutcome(await runTestImpl(testOptions));
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
|
|
@ -93,7 +101,8 @@ const PromptCachingTestCard: React.FC<PromptCachingTestCardProps> = ({
|
|||
<CardTitle>Test prompt caching</CardTitle>
|
||||
<p className="mt-1 break-words text-xs text-muted-foreground">
|
||||
Sends two identical requests with a ~9k token system prompt and no cache_control of its own. If LiteLLM
|
||||
injects the breakpoints, the first call writes the cache and the second reads it.
|
||||
injects the breakpoints, the first call writes the cache and the second reads it. Both calls hit the real
|
||||
model through the proxy and are billed normally.
|
||||
</p>
|
||||
|
||||
{!enabled && (
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ describe("runPromptCachingTest", () => {
|
|||
model: "claude-haiku-4-5",
|
||||
baseUrl: "http://proxy:4000",
|
||||
fetchImpl,
|
||||
customHeaders: { "x-custom-auth": "abc", Authorization: "Bearer wrong" },
|
||||
};
|
||||
const result = await runPromptCachingTest(testOptions);
|
||||
|
||||
|
|
@ -96,6 +97,7 @@ describe("runPromptCachingTest", () => {
|
|||
const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit];
|
||||
expect(url).toBe("http://proxy:4000/v1/chat/completions");
|
||||
expect((init.headers as Record<string, string>).Authorization).toBe("Bearer token-123");
|
||||
expect((init.headers as Record<string, string>)["x-custom-auth"]).toBe("abc");
|
||||
expect((init.headers as Record<string, string>)["x-litellm-tags"]).toBe("prompt-caching-test");
|
||||
|
||||
const serialized = JSON.stringify(seenBodies);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { z } from "zod";
|
||||
|
||||
import { buildPlaygroundHeaders } from "@/components/llm_calls/request_headers";
|
||||
import { CustomHeaders, buildPlaygroundHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers";
|
||||
import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
|
||||
|
||||
export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096;
|
||||
|
|
@ -93,20 +93,29 @@ const parseResponseCost = (headers: Headers): number | null => {
|
|||
return Number.isFinite(cost) ? cost : null;
|
||||
};
|
||||
|
||||
const sendTestCall = async (
|
||||
accessToken: string,
|
||||
baseUrl: string,
|
||||
body: PromptCachingTestRequestBody,
|
||||
fetchImpl: typeof fetch,
|
||||
): Promise<PromptCachingCallResult> => {
|
||||
interface SendTestCallOptions {
|
||||
readonly accessToken: string;
|
||||
readonly baseUrl: string;
|
||||
readonly body: PromptCachingTestRequestBody;
|
||||
readonly fetchImpl: typeof fetch;
|
||||
readonly customHeaders: CustomHeaders;
|
||||
}
|
||||
|
||||
const sendTestCall = async ({
|
||||
accessToken,
|
||||
baseUrl,
|
||||
body,
|
||||
fetchImpl,
|
||||
customHeaders,
|
||||
}: SendTestCallOptions): Promise<PromptCachingCallResult> => {
|
||||
const startedAt = Date.now();
|
||||
const headers = withRequiredHeaders(buildPlaygroundHeaders(["prompt-caching-test"], customHeaders), {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
...buildPlaygroundHeaders(["prompt-caching-test"]),
|
||||
},
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const durationMs = Date.now() - startedAt;
|
||||
|
|
@ -137,6 +146,7 @@ export interface RunPromptCachingTestOptions {
|
|||
readonly baseUrl: string;
|
||||
readonly fetchImpl?: typeof fetch;
|
||||
readonly onCallStart?: (callIndex: 1 | 2) => void;
|
||||
readonly customHeaders?: CustomHeaders;
|
||||
}
|
||||
|
||||
export const runPromptCachingTest = async ({
|
||||
|
|
@ -145,6 +155,7 @@ export const runPromptCachingTest = async ({
|
|||
baseUrl,
|
||||
fetchImpl = fetch,
|
||||
onCallStart,
|
||||
customHeaders = {},
|
||||
}: RunPromptCachingTestOptions): Promise<{
|
||||
first: PromptCachingCallResult;
|
||||
second: PromptCachingCallResult;
|
||||
|
|
@ -153,10 +164,12 @@ export const runPromptCachingTest = async ({
|
|||
const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
const body = buildTestRequestBody(model, systemPrompt);
|
||||
|
||||
const callOptions = { accessToken, baseUrl, body, fetchImpl, customHeaders };
|
||||
|
||||
onCallStart?.(1);
|
||||
const first = await sendTestCall(accessToken, baseUrl, body, fetchImpl);
|
||||
const first = await sendTestCall(callOptions);
|
||||
onCallStart?.(2);
|
||||
const second = await sendTestCall(accessToken, baseUrl, body, fetchImpl);
|
||||
const second = await sendTestCall(callOptions);
|
||||
|
||||
return { first, second, verdict: judgePromptCachingTest(first, second) };
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue