mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 1e891e486e into 0e252c2de3
This commit is contained in:
commit
0b14fbd7e0
10 changed files with 661 additions and 19 deletions
|
|
@ -210,6 +210,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
|
|||
"x-litellm-guardrail-scan-id",
|
||||
"x-litellm-guardrail-scan-metadata",
|
||||
"x-litellm-cache-key",
|
||||
"x-litellm-response-cost",
|
||||
]
|
||||
|
||||
# Gemini model-specific minimal thinking budget constants
|
||||
|
|
|
|||
|
|
@ -103,6 +103,13 @@ def test_cors_exposes_cache_key_header_to_browser_js():
|
|||
assert "x-litellm-cache-key" in cors_middleware.kwargs["expose_headers"]
|
||||
|
||||
|
||||
def test_cors_exposes_response_cost_header_to_browser_js():
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
cors_middleware = next(m for m in app.user_middleware if m.cls is CORSMiddleware)
|
||||
assert "x-litellm-response-cost" in cors_middleware.kwargs["expose_headers"]
|
||||
|
||||
|
||||
def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch):
|
||||
mock_login_result = {"user_id": "test-user"}
|
||||
mock_prisma_client = MagicMock()
|
||||
|
|
|
|||
|
|
@ -37,9 +37,14 @@ vi.mock("@/components/shared/charts", () => ({
|
|||
SEQUENTIAL_COLOR_RAMP: ["indigo"],
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({
|
||||
PromptCachingPanel: () => <div data-testid="caching-settings" />,
|
||||
}));
|
||||
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => {
|
||||
const original =
|
||||
await importOriginal<typeof import("@/app/(dashboard)/router-settings/_components/general_settings")>();
|
||||
return {
|
||||
...original,
|
||||
PromptCachingPanel: () => <div data-testid="caching-settings" />,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () => <div /> }));
|
||||
|
||||
|
|
|
|||
|
|
@ -7,11 +7,17 @@ vi.mock("@/components/networking", () => ({
|
|||
getGeneralSettingsCall: (...args: unknown[]) => mockGetGeneralSettingsCall(...args),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({
|
||||
PromptCachingPanel: () => <div data-testid="caching-settings" />,
|
||||
}));
|
||||
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => {
|
||||
const original =
|
||||
await importOriginal<typeof import("@/app/(dashboard)/router-settings/_components/general_settings")>();
|
||||
return {
|
||||
...original,
|
||||
PromptCachingPanel: () => <div data-testid="caching-settings" />,
|
||||
};
|
||||
});
|
||||
|
||||
const mockCacheLeakageCard = vi.fn();
|
||||
const mockPromptCachingTestCard = vi.fn();
|
||||
|
||||
vi.mock("./CacheLeakageCard", () => ({
|
||||
__esModule: true,
|
||||
|
|
@ -21,27 +27,72 @@ vi.mock("./CacheLeakageCard", () => ({
|
|||
},
|
||||
}));
|
||||
|
||||
vi.mock("./PromptCachingTestCard", () => ({
|
||||
__esModule: true,
|
||||
default: (props: unknown) => {
|
||||
mockPromptCachingTestCard(props);
|
||||
return <div data-testid="prompt-caching-test-card" />;
|
||||
},
|
||||
}));
|
||||
|
||||
import PromptCachingTab from "./PromptCachingTab";
|
||||
|
||||
const activity = {
|
||||
dateValue: {},
|
||||
onDateChange: vi.fn(),
|
||||
results: [],
|
||||
loading: false,
|
||||
isFetchingMore: false,
|
||||
progress: { currentPage: 1, totalPages: 1 },
|
||||
cancelled: false,
|
||||
failed: false,
|
||||
cancel: vi.fn(),
|
||||
};
|
||||
|
||||
describe("PromptCachingTab", () => {
|
||||
it("renders the cache leakage table alongside the caching settings", async () => {
|
||||
mockGetGeneralSettingsCall.mockResolvedValue([]);
|
||||
|
||||
const activity = {
|
||||
dateValue: {},
|
||||
onDateChange: vi.fn(),
|
||||
results: [],
|
||||
loading: false,
|
||||
isFetchingMore: false,
|
||||
progress: { currentPage: 1, totalPages: 1 },
|
||||
cancelled: false,
|
||||
failed: false,
|
||||
cancel: vi.fn(),
|
||||
};
|
||||
render(<PromptCachingTab accessToken="test-token" activity={activity} />);
|
||||
|
||||
expect(screen.getByTestId("caching-settings")).toBeInTheDocument();
|
||||
expect(screen.getByTestId("cache-leakage-card")).toBeInTheDocument();
|
||||
await waitFor(() => expect(mockCacheLeakageCard).toHaveBeenCalledWith(expect.objectContaining({ activity })));
|
||||
});
|
||||
|
||||
it("passes enabled=true to the test card when the caching setting is on", async () => {
|
||||
mockGetGeneralSettingsCall.mockResolvedValue([
|
||||
{
|
||||
field_name: "enable_anthropic_prompt_caching",
|
||||
field_type: "Boolean",
|
||||
field_value: true,
|
||||
field_description: "",
|
||||
stored_in_db: true,
|
||||
},
|
||||
]);
|
||||
|
||||
render(<PromptCachingTab accessToken="test-token" activity={activity} />);
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: true })),
|
||||
);
|
||||
});
|
||||
|
||||
it("passes enabled=false to the test card when the caching setting is off", async () => {
|
||||
mockGetGeneralSettingsCall.mockResolvedValue([
|
||||
{
|
||||
field_name: "enable_anthropic_prompt_caching",
|
||||
field_type: "Boolean",
|
||||
field_value: false,
|
||||
field_description: "",
|
||||
stored_in_db: true,
|
||||
},
|
||||
]);
|
||||
|
||||
render(<PromptCachingTab accessToken="test-token" activity={activity} />);
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: false })),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@ import React, { useCallback, useEffect, useState } from "react";
|
|||
import { getGeneralSettingsCall } from "@/components/networking";
|
||||
import { toast } from "@/lib/toast";
|
||||
import {
|
||||
ENABLE_ANTHROPIC_PROMPT_CACHING,
|
||||
PromptCachingPanel,
|
||||
generalSettingsItem,
|
||||
isOn,
|
||||
} from "@/app/(dashboard)/router-settings/_components/general_settings";
|
||||
import CacheLeakageCard from "./CacheLeakageCard";
|
||||
import PromptCachingTestCard from "./PromptCachingTestCard";
|
||||
import { DailyActivityRange } from "./useDailyActivityRange";
|
||||
|
||||
interface PromptCachingTabProps {
|
||||
|
|
@ -45,9 +48,12 @@ const PromptCachingTab: React.FC<PromptCachingTabProps> = ({ accessToken, activi
|
|||
return null;
|
||||
}
|
||||
|
||||
const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING);
|
||||
|
||||
return (
|
||||
<div className="w-full space-y-6">
|
||||
<PromptCachingPanel accessToken={accessToken} settings={settings} onChange={handleChange} />
|
||||
<PromptCachingTestCard accessToken={accessToken} enabled={isOn(enableSetting?.field_value)} />
|
||||
<CacheLeakageCard activity={activity} />
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
import { fireEvent, render, screen } from "@testing-library/react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getProxyBaseUrl: vi.fn(() => "http://proxy:4000"),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/common_components/ModelSelector", () => ({
|
||||
__esModule: true,
|
||||
default: ({ onChange }: { onChange?: (value: string | null) => void }) => (
|
||||
<button data-testid="model-selector" onClick={() => onChange?.("claude-haiku-4-5")} />
|
||||
),
|
||||
}));
|
||||
|
||||
import PromptCachingTestCard from "./PromptCachingTestCard";
|
||||
|
||||
const successOutcome = {
|
||||
first: {
|
||||
cacheCreationTokens: 5000,
|
||||
cacheReadTokens: 0,
|
||||
promptTokens: 5010,
|
||||
responseCost: 0.004,
|
||||
model: "claude-haiku-4-5",
|
||||
durationMs: 120,
|
||||
},
|
||||
second: {
|
||||
cacheCreationTokens: 0,
|
||||
cacheReadTokens: 5000,
|
||||
promptTokens: 5010,
|
||||
responseCost: 0.001,
|
||||
model: "claude-haiku-4-5",
|
||||
durationMs: 80,
|
||||
},
|
||||
verdict: "injected" as const,
|
||||
};
|
||||
|
||||
describe("PromptCachingTestCard", () => {
|
||||
it("keeps the run button disabled until a model is picked", () => {
|
||||
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={vi.fn()} />);
|
||||
|
||||
expect(screen.getByRole("button", { name: /run test/i })).toBeDisabled();
|
||||
});
|
||||
|
||||
it("shows the injected verdict after a successful run", async () => {
|
||||
const runTest = vi.fn(async () => successOutcome);
|
||||
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={runTest} />);
|
||||
|
||||
fireEvent.click(screen.getByTestId("model-selector"));
|
||||
fireEvent.click(screen.getByRole("button", { name: /run test/i }));
|
||||
|
||||
expect(await screen.findByText(/LiteLLM injected cache_control: call 1 wrote 5000 tokens/)).toBeInTheDocument();
|
||||
expect(runTest).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ accessToken: "token", model: "claude-haiku-4-5", baseUrl: "http://proxy:4000" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the injected_no_read verdict when call 2 misses the cache", async () => {
|
||||
const runTest = vi.fn(async () => ({
|
||||
...successOutcome,
|
||||
second: { ...successOutcome.second, cacheReadTokens: 0 },
|
||||
verdict: "injected_no_read" as const,
|
||||
}));
|
||||
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={runTest} />);
|
||||
|
||||
fireEvent.click(screen.getByTestId("model-selector"));
|
||||
fireEvent.click(screen.getByRole("button", { name: /run test/i }));
|
||||
|
||||
expect(await screen.findByText(/cache_control injected, but the second call missed the cache/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/Call 1 wrote 5000 tokens to the cache but call 2 read 0/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows an error alert when the run throws", async () => {
|
||||
const runTest = vi.fn(async () => {
|
||||
throw new Error("Request failed with status 400: boom");
|
||||
});
|
||||
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={runTest} />);
|
||||
|
||||
fireEvent.click(screen.getByTestId("model-selector"));
|
||||
fireEvent.click(screen.getByRole("button", { name: /run test/i }));
|
||||
|
||||
expect(await screen.findByText(/status 400: boom/)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,185 @@
|
|||
"use client";
|
||||
|
||||
import React, { useState } from "react";
|
||||
import { CircleAlert, CircleCheck, CirclePlay } from "lucide-react";
|
||||
|
||||
import ModelSelector from "@/components/common_components/ModelSelector";
|
||||
import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers";
|
||||
import { getProxyBaseUrl } from "@/components/networking";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardTitle } from "@/components/ui/card";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { getSecureItem } from "@/utils/secureStorage";
|
||||
import { PromptCachingCallResult, PromptCachingVerdict, runPromptCachingTest } from "./promptCachingTest";
|
||||
|
||||
interface PromptCachingTestCardProps {
|
||||
accessToken: string;
|
||||
enabled: boolean;
|
||||
runTest?: typeof runPromptCachingTest;
|
||||
}
|
||||
|
||||
interface TestOutcome {
|
||||
first: PromptCachingCallResult;
|
||||
second: PromptCachingCallResult;
|
||||
verdict: PromptCachingVerdict;
|
||||
}
|
||||
|
||||
const formatCost = (cost: number | null) => (cost === null ? "-" : `$${cost.toFixed(6)}`);
|
||||
|
||||
const VERDICT_COPY: Record<
|
||||
PromptCachingVerdict,
|
||||
{ variant: "success" | "info" | "warning" | "error"; title?: string; text: (o: TestOutcome) => string }
|
||||
> = {
|
||||
injected: {
|
||||
variant: "success",
|
||||
text: (o) =>
|
||||
`LiteLLM injected cache_control: call 1 wrote ${o.first.cacheCreationTokens} tokens to the cache and call 2 read ${o.second.cacheReadTokens} tokens.`,
|
||||
},
|
||||
injected_no_read: {
|
||||
variant: "warning",
|
||||
title: "cache_control injected, but the second call missed the cache",
|
||||
text: (o) =>
|
||||
`Call 1 wrote ${o.first.cacheCreationTokens} tokens to the cache but call 2 read 0. LiteLLM is injecting the breakpoints; check the cache TTL or whether both calls reached the same upstream credentials.`,
|
||||
},
|
||||
cache_hit_only: {
|
||||
variant: "info",
|
||||
text: () => "Cache read on both calls: cache_control was on the wire and a warm cache already existed.",
|
||||
},
|
||||
not_injected: {
|
||||
variant: "error",
|
||||
text: () =>
|
||||
"No cache activity. LiteLLM did not inject cache_control for this model. Check the toggle above, that the model is an Anthropic or Bedrock Claude model with prompt caching support, and that the key does not disable it.",
|
||||
},
|
||||
};
|
||||
|
||||
const CallRow = ({ label, call }: { label: string; call: PromptCachingCallResult }) => (
|
||||
<TableRow>
|
||||
<TableCell className="font-medium">{label}</TableCell>
|
||||
<TableCell>{call.promptTokens}</TableCell>
|
||||
<TableCell>{call.cacheCreationTokens}</TableCell>
|
||||
<TableCell>{call.cacheReadTokens}</TableCell>
|
||||
<TableCell>{formatCost(call.responseCost)}</TableCell>
|
||||
<TableCell>{call.durationMs} ms</TableCell>
|
||||
</TableRow>
|
||||
);
|
||||
|
||||
const PromptCachingTestCard: React.FC<PromptCachingTestCardProps> = ({
|
||||
accessToken,
|
||||
enabled,
|
||||
runTest: runTestImpl = runPromptCachingTest,
|
||||
}) => {
|
||||
const [model, setModel] = useState<string | null>(null);
|
||||
const [runningCall, setRunningCall] = useState<0 | 1 | 2>(0);
|
||||
const [outcome, setOutcome] = useState<TestOutcome | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const running = runningCall !== 0;
|
||||
|
||||
const handleRunTest = async () => {
|
||||
if (!model || running) {
|
||||
return;
|
||||
}
|
||||
setRunningCall(1);
|
||||
setOutcome(null);
|
||||
setError(null);
|
||||
try {
|
||||
const testOptions = {
|
||||
accessToken,
|
||||
model,
|
||||
baseUrl: getProxyBaseUrl(),
|
||||
onCallStart: setRunningCall,
|
||||
customHeaders: customHeadersFromPairs(parseStoredHeaderPairs(getSecureItem("customHeaders"))),
|
||||
};
|
||||
setOutcome(await runTestImpl(testOptions));
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setRunningCall(0);
|
||||
}
|
||||
};
|
||||
|
||||
const verdict = outcome ? VERDICT_COPY[outcome.verdict] : null;
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardContent>
|
||||
<CardTitle>Test prompt caching</CardTitle>
|
||||
<p className="mt-1 break-words text-xs text-muted-foreground">
|
||||
Sends two identical requests with a ~9k token system prompt and no cache_control of its own. If LiteLLM
|
||||
injects the breakpoints, the first call writes the cache and the second reads it. Both calls hit the real
|
||||
model through the proxy and are billed normally.
|
||||
</p>
|
||||
|
||||
{!enabled && (
|
||||
<Alert variant="info" className="mt-4">
|
||||
<CircleAlert />
|
||||
<AlertTitle>Automatic prompt caching is off</AlertTitle>
|
||||
<AlertDescription>
|
||||
Turn it on above to test injection, or run anyway to confirm nothing is injected.
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
<div className="mt-4 flex flex-wrap items-end gap-4">
|
||||
<div className="min-w-64">
|
||||
<ModelSelector
|
||||
accessToken={accessToken}
|
||||
value={model}
|
||||
onChange={setModel}
|
||||
disabled={running}
|
||||
labelText="Model"
|
||||
/>
|
||||
</div>
|
||||
<Button onClick={handleRunTest} disabled={running || !model}>
|
||||
<CirclePlay />
|
||||
Run test
|
||||
</Button>
|
||||
{running && <span className="text-sm text-muted-foreground">Sending call {runningCall} of 2...</span>}
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<Alert variant="error" className="mt-4">
|
||||
<CircleAlert />
|
||||
<AlertTitle>Test failed</AlertTitle>
|
||||
<AlertDescription>{error}</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{outcome && (
|
||||
<div className="mt-4">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead />
|
||||
<TableHead>Prompt tokens</TableHead>
|
||||
<TableHead>Cache write</TableHead>
|
||||
<TableHead>Cache read</TableHead>
|
||||
<TableHead>Cost</TableHead>
|
||||
<TableHead>Latency</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
<CallRow label="Call 1" call={outcome.first} />
|
||||
<CallRow label="Call 2" call={outcome.second} />
|
||||
</TableBody>
|
||||
</Table>
|
||||
{verdict && (
|
||||
<Alert variant={verdict.variant} className="mt-4">
|
||||
{outcome.verdict === "injected" || outcome.verdict === "cache_hit_only" ? (
|
||||
<CircleCheck />
|
||||
) : (
|
||||
<CircleAlert />
|
||||
)}
|
||||
{verdict.title && <AlertTitle>{verdict.title}</AlertTitle>}
|
||||
<AlertDescription>{verdict.text(outcome)}</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
export default PromptCachingTestCard;
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS,
|
||||
PromptCachingCallResult,
|
||||
buildTestSystemPrompt,
|
||||
judgePromptCachingTest,
|
||||
runPromptCachingTest,
|
||||
} from "./promptCachingTest";
|
||||
|
||||
const callResult = (overrides: Partial<PromptCachingCallResult> = {}): PromptCachingCallResult => ({
|
||||
cacheCreationTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
promptTokens: 5000,
|
||||
responseCost: null,
|
||||
model: null,
|
||||
durationMs: 10,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const jsonResponse = (body: unknown, init: { status?: number; headers?: Record<string, string> } = {}) =>
|
||||
new Response(JSON.stringify(body), {
|
||||
status: init.status ?? 200,
|
||||
headers: { "Content-Type": "application/json", ...init.headers },
|
||||
});
|
||||
|
||||
const anthropicUsage = {
|
||||
prompt_tokens: 5010,
|
||||
cache_creation_input_tokens: 5000,
|
||||
cache_read_input_tokens: 0,
|
||||
};
|
||||
|
||||
describe("buildTestSystemPrompt", () => {
|
||||
it("is comfortably above Anthropic's largest minimum cacheable size and differs by nonce", () => {
|
||||
const first = buildTestSystemPrompt("run-a");
|
||||
const second = buildTestSystemPrompt("run-b");
|
||||
|
||||
expect(first.length).toBeGreaterThan(20000);
|
||||
expect(first.length).toBeGreaterThan(PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS * 4);
|
||||
expect(first).not.toBe(second);
|
||||
expect(first.startsWith("Test run run-a.")).toBe(true);
|
||||
expect(first.slice(first.indexOf("\n"))).toBe(second.slice(second.indexOf("\n")));
|
||||
});
|
||||
});
|
||||
|
||||
describe("judgePromptCachingTest", () => {
|
||||
it("returns injected when call 1 writes and call 2 reads", () => {
|
||||
expect(
|
||||
judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult({ cacheReadTokens: 5000 })),
|
||||
).toBe("injected");
|
||||
});
|
||||
|
||||
it("returns cache_hit_only when neither call writes but a call reads", () => {
|
||||
expect(judgePromptCachingTest(callResult(), callResult({ cacheReadTokens: 5000 }))).toBe("cache_hit_only");
|
||||
expect(judgePromptCachingTest(callResult({ cacheReadTokens: 5000 }), callResult({ cacheReadTokens: 5000 }))).toBe(
|
||||
"cache_hit_only",
|
||||
);
|
||||
});
|
||||
|
||||
it("returns injected_no_read when call 1 writes but call 2 reads nothing", () => {
|
||||
expect(judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult())).toBe("injected_no_read");
|
||||
});
|
||||
|
||||
it("returns not_injected with no cache activity at all", () => {
|
||||
expect(judgePromptCachingTest(callResult(), callResult())).toBe("not_injected");
|
||||
});
|
||||
});
|
||||
|
||||
describe("runPromptCachingTest", () => {
|
||||
it("sends two sequential identical requests with no cache_control and parses usage", async () => {
|
||||
const seenBodies: unknown[] = [];
|
||||
const responses = [
|
||||
jsonResponse(
|
||||
{ model: "claude-haiku-4-5", usage: anthropicUsage },
|
||||
{ headers: { "x-litellm-response-cost": "0.0042" } },
|
||||
),
|
||||
jsonResponse({
|
||||
model: "claude-haiku-4-5",
|
||||
usage: { prompt_tokens: 5010, prompt_tokens_details: { cached_tokens: 5000 } },
|
||||
}),
|
||||
];
|
||||
const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => {
|
||||
seenBodies.push(JSON.parse(String(init?.body)));
|
||||
return responses[seenBodies.length - 1];
|
||||
});
|
||||
|
||||
const testOptions = {
|
||||
accessToken: "token-123",
|
||||
model: "claude-haiku-4-5",
|
||||
baseUrl: "http://proxy:4000",
|
||||
fetchImpl,
|
||||
customHeaders: { "x-custom-auth": "abc", Authorization: "Bearer wrong" },
|
||||
};
|
||||
const result = await runPromptCachingTest(testOptions);
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit];
|
||||
expect(url).toBe("http://proxy:4000/v1/chat/completions");
|
||||
expect((init.headers as Record<string, string>).Authorization).toBe("Bearer token-123");
|
||||
expect((init.headers as Record<string, string>)["x-custom-auth"]).toBe("abc");
|
||||
expect((init.headers as Record<string, string>)["x-litellm-tags"]).toBe("prompt-caching-test");
|
||||
|
||||
const serialized = JSON.stringify(seenBodies);
|
||||
expect(serialized).not.toContain("cache_control");
|
||||
expect((seenBodies[0] as { messages: { content: string }[] }).messages[0].content).toBe(
|
||||
(seenBodies[1] as { messages: { content: string }[] }).messages[0].content,
|
||||
);
|
||||
|
||||
expect(result.first.cacheCreationTokens).toBe(5000);
|
||||
expect(result.first.responseCost).toBeCloseTo(0.0042);
|
||||
expect(result.second.cacheReadTokens).toBe(5000);
|
||||
expect(result.verdict).toBe("injected");
|
||||
});
|
||||
|
||||
it("rejects with status and error message on non-2xx", async () => {
|
||||
const fetchImpl = vi.fn(async () => jsonResponse({ error: { message: "boom" } }, { status: 400 }));
|
||||
|
||||
const testOptions = {
|
||||
accessToken: "token-123",
|
||||
model: "claude-haiku-4-5",
|
||||
baseUrl: "http://proxy:4000",
|
||||
fetchImpl,
|
||||
};
|
||||
await expect(runPromptCachingTest(testOptions)).rejects.toThrow(/400.*boom|boom.*400/);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
import { z } from "zod";
|
||||
|
||||
import { CustomHeaders, buildPlaygroundHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers";
|
||||
import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
|
||||
|
||||
export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096;
|
||||
|
||||
const FILLER_LINE_COUNT = 600;
|
||||
|
||||
export const buildTestSystemPrompt = (nonce: string): string => {
|
||||
const filler = Array.from(
|
||||
{ length: FILLER_LINE_COUNT },
|
||||
(_, index) => `Reference item ${index + 1}: the gateway forwards requests to the configured deployment.`,
|
||||
).join("\n");
|
||||
return `Test run ${nonce}.\n${filler}`;
|
||||
};
|
||||
|
||||
export interface PromptCachingTestRequestBody {
|
||||
readonly model: string;
|
||||
readonly messages: readonly { readonly role: string; readonly content: string }[];
|
||||
readonly max_tokens: number;
|
||||
readonly temperature: number;
|
||||
}
|
||||
|
||||
export const buildTestRequestBody = (model: string, systemPrompt: string): PromptCachingTestRequestBody => ({
|
||||
model,
|
||||
messages: [
|
||||
{ role: "system", content: systemPrompt },
|
||||
{ role: "user", content: "Reply with the single word OK." },
|
||||
],
|
||||
max_tokens: 5,
|
||||
temperature: 0,
|
||||
});
|
||||
|
||||
export interface PromptCachingCallResult {
|
||||
cacheCreationTokens: number;
|
||||
cacheReadTokens: number;
|
||||
promptTokens: number;
|
||||
responseCost: number | null;
|
||||
model: string | null;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export type PromptCachingVerdict = "injected" | "injected_no_read" | "not_injected" | "cache_hit_only";
|
||||
|
||||
export const judgePromptCachingTest = (
|
||||
first: PromptCachingCallResult,
|
||||
second: PromptCachingCallResult,
|
||||
): PromptCachingVerdict => {
|
||||
if (first.cacheCreationTokens > 0 && second.cacheReadTokens > 0) {
|
||||
return "injected";
|
||||
}
|
||||
if (first.cacheCreationTokens > 0 && second.cacheReadTokens === 0) {
|
||||
return "injected_no_read";
|
||||
}
|
||||
if (first.cacheCreationTokens === 0 && second.cacheCreationTokens === 0) {
|
||||
if (first.cacheReadTokens > 0 || second.cacheReadTokens > 0) {
|
||||
return "cache_hit_only";
|
||||
}
|
||||
}
|
||||
return "not_injected";
|
||||
};
|
||||
|
||||
const tokenDetailsSchema = z
|
||||
.object({
|
||||
cached_tokens: z.number().nullish(),
|
||||
cache_write_tokens: z.number().nullish(),
|
||||
})
|
||||
.nullish();
|
||||
|
||||
const usageFields = {
|
||||
prompt_tokens: z.number().nullish(),
|
||||
cache_creation_input_tokens: z.number().nullish(),
|
||||
cache_read_input_tokens: z.number().nullish(),
|
||||
prompt_tokens_details: tokenDetailsSchema,
|
||||
input_tokens_details: tokenDetailsSchema,
|
||||
};
|
||||
|
||||
const usageSchema = z.object(usageFields).nullish();
|
||||
|
||||
const chatCompletionResponseSchema = z.object({
|
||||
model: z.string().nullish(),
|
||||
usage: usageSchema,
|
||||
});
|
||||
|
||||
const errorBodySchema = z.object({
|
||||
error: z.object({ message: z.string().optional() }).optional(),
|
||||
});
|
||||
|
||||
const parseResponseCost = (headers: Headers): number | null => {
|
||||
const raw = headers.get("x-litellm-response-cost");
|
||||
if (raw === null) {
|
||||
return null;
|
||||
}
|
||||
const cost = Number.parseFloat(raw);
|
||||
return Number.isFinite(cost) ? cost : null;
|
||||
};
|
||||
|
||||
interface SendTestCallOptions {
|
||||
readonly accessToken: string;
|
||||
readonly baseUrl: string;
|
||||
readonly body: PromptCachingTestRequestBody;
|
||||
readonly fetchImpl: typeof fetch;
|
||||
readonly customHeaders: CustomHeaders;
|
||||
}
|
||||
|
||||
const sendTestCall = async ({
|
||||
accessToken,
|
||||
baseUrl,
|
||||
body,
|
||||
fetchImpl,
|
||||
customHeaders,
|
||||
}: SendTestCallOptions): Promise<PromptCachingCallResult> => {
|
||||
const startedAt = Date.now();
|
||||
const headers = withRequiredHeaders(buildPlaygroundHeaders(["prompt-caching-test"], customHeaders), {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const durationMs = Date.now() - startedAt;
|
||||
|
||||
const json: unknown = await response.json();
|
||||
if (!response.ok) {
|
||||
const parsedError = errorBodySchema.safeParse(json);
|
||||
const detail = parsedError.success ? parsedError.data.error?.message : undefined;
|
||||
throw new Error(`Request failed with status ${response.status}${detail ? `: ${detail}` : ""}`);
|
||||
}
|
||||
|
||||
const parsed = chatCompletionResponseSchema.parse(json);
|
||||
const cacheTokens = extractPromptCacheTokens(parsed.usage);
|
||||
|
||||
return {
|
||||
cacheCreationTokens: cacheTokens.cacheCreationTokens ?? 0,
|
||||
cacheReadTokens: cacheTokens.cacheReadTokens ?? 0,
|
||||
promptTokens: parsed.usage?.prompt_tokens ?? 0,
|
||||
responseCost: parseResponseCost(response.headers),
|
||||
model: parsed.model ?? null,
|
||||
durationMs,
|
||||
};
|
||||
};
|
||||
|
||||
export interface RunPromptCachingTestOptions {
|
||||
readonly accessToken: string;
|
||||
readonly model: string;
|
||||
readonly baseUrl: string;
|
||||
readonly fetchImpl?: typeof fetch;
|
||||
readonly onCallStart?: (callIndex: 1 | 2) => void;
|
||||
readonly customHeaders?: CustomHeaders;
|
||||
}
|
||||
|
||||
export const runPromptCachingTest = async ({
|
||||
accessToken,
|
||||
model,
|
||||
baseUrl,
|
||||
fetchImpl = fetch,
|
||||
onCallStart,
|
||||
customHeaders = {},
|
||||
}: RunPromptCachingTestOptions): Promise<{
|
||||
first: PromptCachingCallResult;
|
||||
second: PromptCachingCallResult;
|
||||
verdict: PromptCachingVerdict;
|
||||
}> => {
|
||||
const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
const body = buildTestRequestBody(model, systemPrompt);
|
||||
|
||||
const callOptions = { accessToken, baseUrl, body, fetchImpl, customHeaders };
|
||||
|
||||
onCallStart?.(1);
|
||||
const first = await sendTestCall(callOptions);
|
||||
onCallStart?.(2);
|
||||
const second = await sendTestCall(callOptions);
|
||||
|
||||
return { first, second, verdict: judgePromptCachingTest(first, second) };
|
||||
};
|
||||
|
|
@ -16,11 +16,11 @@ import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"
|
|||
import RoutingGroups from "@/components/routing_groups";
|
||||
|
||||
const PROMPT_CACHING_TAB = "prompt_caching";
|
||||
const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching";
|
||||
export const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching";
|
||||
const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl";
|
||||
const OPENAI_SYSTEM_MESSAGES_FIRST = "openai_system_messages_first";
|
||||
|
||||
const isOn = (value: unknown) => value === true || value === "true";
|
||||
export const isOn = (value: unknown) => value === true || value === "true";
|
||||
|
||||
interface GeneralSettingsPageProps {
|
||||
accessToken: string | null;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue