This commit is contained in:
devin-ai-integration[bot] 2026-09-21 11:32:32 -04:00 • committed by GitHub
commit 0b14fbd7e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 661 additions and 19 deletions

View file

@ -210,6 +210,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
"x-litellm-guardrail-scan-id",
"x-litellm-guardrail-scan-metadata",
"x-litellm-cache-key",
"x-litellm-response-cost",
]
# Gemini model-specific minimal thinking budget constants

View file

@ -103,6 +103,13 @@ def test_cors_exposes_cache_key_header_to_browser_js():
assert "x-litellm-cache-key" in cors_middleware.kwargs["expose_headers"]
def test_cors_exposes_response_cost_header_to_browser_js():
from fastapi.middleware.cors import CORSMiddleware
cors_middleware = next(m for m in app.user_middleware if m.cls is CORSMiddleware)
assert "x-litellm-response-cost" in cors_middleware.kwargs["expose_headers"]
def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch):
mock_login_result = {"user_id": "test-user"}
mock_prisma_client = MagicMock()

View file

@ -37,9 +37,14 @@ vi.mock("@/components/shared/charts", () => ({
SEQUENTIAL_COLOR_RAMP: ["indigo"],
}));
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({
PromptCachingPanel: () => <div data-testid="caching-settings" />,
}));
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => {
const original =
await importOriginal<typeof import("@/app/(dashboard)/router-settings/_components/general_settings")>();
return {
...original,
PromptCachingPanel: () => <div data-testid="caching-settings" />,
};
});
vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () => <div /> }));

View file

@ -7,11 +7,17 @@ vi.mock("@/components/networking", () => ({
getGeneralSettingsCall: (...args: unknown[]) => mockGetGeneralSettingsCall(...args),
}));
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({
PromptCachingPanel: () => <div data-testid="caching-settings" />,
}));
vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", async (importOriginal) => {
const original =
await importOriginal<typeof import("@/app/(dashboard)/router-settings/_components/general_settings")>();
return {
...original,
PromptCachingPanel: () => <div data-testid="caching-settings" />,
};
});
const mockCacheLeakageCard = vi.fn();
const mockPromptCachingTestCard = vi.fn();
vi.mock("./CacheLeakageCard", () => ({
__esModule: true,
@ -21,27 +27,72 @@ vi.mock("./CacheLeakageCard", () => ({
},
}));
vi.mock("./PromptCachingTestCard", () => ({
__esModule: true,
default: (props: unknown) => {
mockPromptCachingTestCard(props);
return <div data-testid="prompt-caching-test-card" />;
},
}));
import PromptCachingTab from "./PromptCachingTab";
const activity = {
dateValue: {},
onDateChange: vi.fn(),
results: [],
loading: false,
isFetchingMore: false,
progress: { currentPage: 1, totalPages: 1 },
cancelled: false,
failed: false,
cancel: vi.fn(),
};
describe("PromptCachingTab", () => {
it("renders the cache leakage table alongside the caching settings", async () => {
mockGetGeneralSettingsCall.mockResolvedValue([]);
const activity = {
dateValue: {},
onDateChange: vi.fn(),
results: [],
loading: false,
isFetchingMore: false,
progress: { currentPage: 1, totalPages: 1 },
cancelled: false,
failed: false,
cancel: vi.fn(),
};
render(<PromptCachingTab accessToken="test-token" activity={activity} />);
expect(screen.getByTestId("caching-settings")).toBeInTheDocument();
expect(screen.getByTestId("cache-leakage-card")).toBeInTheDocument();
await waitFor(() => expect(mockCacheLeakageCard).toHaveBeenCalledWith(expect.objectContaining({ activity })));
});
it("passes enabled=true to the test card when the caching setting is on", async () => {
mockGetGeneralSettingsCall.mockResolvedValue([
{
field_name: "enable_anthropic_prompt_caching",
field_type: "Boolean",
field_value: true,
field_description: "",
stored_in_db: true,
},
]);
render(<PromptCachingTab accessToken="test-token" activity={activity} />);
await waitFor(() =>
expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: true })),
);
});
it("passes enabled=false to the test card when the caching setting is off", async () => {
mockGetGeneralSettingsCall.mockResolvedValue([
{
field_name: "enable_anthropic_prompt_caching",
field_type: "Boolean",
field_value: false,
field_description: "",
stored_in_db: true,
},
]);
render(<PromptCachingTab accessToken="test-token" activity={activity} />);
await waitFor(() =>
expect(mockPromptCachingTestCard).toHaveBeenCalledWith(expect.objectContaining({ enabled: false })),
);
});
});

View file

@ -5,10 +5,13 @@ import React, { useCallback, useEffect, useState } from "react";
import { getGeneralSettingsCall } from "@/components/networking";
import { toast } from "@/lib/toast";
import {
ENABLE_ANTHROPIC_PROMPT_CACHING,
PromptCachingPanel,
generalSettingsItem,
isOn,
} from "@/app/(dashboard)/router-settings/_components/general_settings";
import CacheLeakageCard from "./CacheLeakageCard";
import PromptCachingTestCard from "./PromptCachingTestCard";
import { DailyActivityRange } from "./useDailyActivityRange";
interface PromptCachingTabProps {
@ -45,9 +48,12 @@ const PromptCachingTab: React.FC<PromptCachingTabProps> = ({ accessToken, activi
return null;
}
const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING);
return (
<div className="w-full space-y-6">
<PromptCachingPanel accessToken={accessToken} settings={settings} onChange={handleChange} />
<PromptCachingTestCard accessToken={accessToken} enabled={isOn(enableSetting?.field_value)} />
<CacheLeakageCard activity={activity} />
</div>
);

View file

@ -0,0 +1,83 @@
import { fireEvent, render, screen } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
vi.mock("@/components/networking", () => ({
getProxyBaseUrl: vi.fn(() => "http://proxy:4000"),
}));
vi.mock("@/components/common_components/ModelSelector", () => ({
__esModule: true,
default: ({ onChange }: { onChange?: (value: string | null) => void }) => (
<button data-testid="model-selector" onClick={() => onChange?.("claude-haiku-4-5")} />
),
}));
import PromptCachingTestCard from "./PromptCachingTestCard";
const successOutcome = {
first: {
cacheCreationTokens: 5000,
cacheReadTokens: 0,
promptTokens: 5010,
responseCost: 0.004,
model: "claude-haiku-4-5",
durationMs: 120,
},
second: {
cacheCreationTokens: 0,
cacheReadTokens: 5000,
promptTokens: 5010,
responseCost: 0.001,
model: "claude-haiku-4-5",
durationMs: 80,
},
verdict: "injected" as const,
};
describe("PromptCachingTestCard", () => {
it("keeps the run button disabled until a model is picked", () => {
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={vi.fn()} />);
expect(screen.getByRole("button", { name: /run test/i })).toBeDisabled();
});
it("shows the injected verdict after a successful run", async () => {
const runTest = vi.fn(async () => successOutcome);
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={runTest} />);
fireEvent.click(screen.getByTestId("model-selector"));
fireEvent.click(screen.getByRole("button", { name: /run test/i }));
expect(await screen.findByText(/LiteLLM injected cache_control: call 1 wrote 5000 tokens/)).toBeInTheDocument();
expect(runTest).toHaveBeenCalledWith(
expect.objectContaining({ accessToken: "token", model: "claude-haiku-4-5", baseUrl: "http://proxy:4000" }),
);
});
it("shows the injected_no_read verdict when call 2 misses the cache", async () => {
const runTest = vi.fn(async () => ({
...successOutcome,
second: { ...successOutcome.second, cacheReadTokens: 0 },
verdict: "injected_no_read" as const,
}));
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={runTest} />);
fireEvent.click(screen.getByTestId("model-selector"));
fireEvent.click(screen.getByRole("button", { name: /run test/i }));
expect(await screen.findByText(/cache_control injected, but the second call missed the cache/)).toBeInTheDocument();
expect(screen.getByText(/Call 1 wrote 5000 tokens to the cache but call 2 read 0/)).toBeInTheDocument();
});
it("shows an error alert when the run throws", async () => {
const runTest = vi.fn(async () => {
throw new Error("Request failed with status 400: boom");
});
render(<PromptCachingTestCard accessToken="token" enabled={true} runTest={runTest} />);
fireEvent.click(screen.getByTestId("model-selector"));
fireEvent.click(screen.getByRole("button", { name: /run test/i }));
expect(await screen.findByText(/status 400: boom/)).toBeInTheDocument();
});
});

View file

@ -0,0 +1,185 @@
"use client";
import React, { useState } from "react";
import { CircleAlert, CircleCheck, CirclePlay } from "lucide-react";
import ModelSelector from "@/components/common_components/ModelSelector";
import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers";
import { getProxyBaseUrl } from "@/components/networking";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardTitle } from "@/components/ui/card";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { getSecureItem } from "@/utils/secureStorage";
import { PromptCachingCallResult, PromptCachingVerdict, runPromptCachingTest } from "./promptCachingTest";
interface PromptCachingTestCardProps {
accessToken: string;
enabled: boolean;
runTest?: typeof runPromptCachingTest;
}
interface TestOutcome {
first: PromptCachingCallResult;
second: PromptCachingCallResult;
verdict: PromptCachingVerdict;
}
const formatCost = (cost: number | null) => (cost === null ? "-" : `$${cost.toFixed(6)}`);
const VERDICT_COPY: Record<
PromptCachingVerdict,
{ variant: "success" | "info" | "warning" | "error"; title?: string; text: (o: TestOutcome) => string }
> = {
injected: {
variant: "success",
text: (o) =>
`LiteLLM injected cache_control: call 1 wrote ${o.first.cacheCreationTokens} tokens to the cache and call 2 read ${o.second.cacheReadTokens} tokens.`,
},
injected_no_read: {
variant: "warning",
title: "cache_control injected, but the second call missed the cache",
text: (o) =>
`Call 1 wrote ${o.first.cacheCreationTokens} tokens to the cache but call 2 read 0. LiteLLM is injecting the breakpoints; check the cache TTL or whether both calls reached the same upstream credentials.`,
},
cache_hit_only: {
variant: "info",
text: () => "Cache read on both calls: cache_control was on the wire and a warm cache already existed.",
},
not_injected: {
variant: "error",
text: () =>
"No cache activity. LiteLLM did not inject cache_control for this model. Check the toggle above, that the model is an Anthropic or Bedrock Claude model with prompt caching support, and that the key does not disable it.",
},
};
const CallRow = ({ label, call }: { label: string; call: PromptCachingCallResult }) => (
<TableRow>
<TableCell className="font-medium">{label}</TableCell>
<TableCell>{call.promptTokens}</TableCell>
<TableCell>{call.cacheCreationTokens}</TableCell>
<TableCell>{call.cacheReadTokens}</TableCell>
<TableCell>{formatCost(call.responseCost)}</TableCell>
<TableCell>{call.durationMs} ms</TableCell>
</TableRow>
);
const PromptCachingTestCard: React.FC<PromptCachingTestCardProps> = ({
accessToken,
enabled,
runTest: runTestImpl = runPromptCachingTest,
}) => {
const [model, setModel] = useState<string | null>(null);
const [runningCall, setRunningCall] = useState<0 | 1 | 2>(0);
const [outcome, setOutcome] = useState<TestOutcome | null>(null);
const [error, setError] = useState<string | null>(null);
const running = runningCall !== 0;
const handleRunTest = async () => {
if (!model || running) {
return;
}
setRunningCall(1);
setOutcome(null);
setError(null);
try {
const testOptions = {
accessToken,
model,
baseUrl: getProxyBaseUrl(),
onCallStart: setRunningCall,
customHeaders: customHeadersFromPairs(parseStoredHeaderPairs(getSecureItem("customHeaders"))),
};
setOutcome(await runTestImpl(testOptions));
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setRunningCall(0);
}
};
const verdict = outcome ? VERDICT_COPY[outcome.verdict] : null;
return (
<Card>
<CardContent>
<CardTitle>Test prompt caching</CardTitle>
<p className="mt-1 break-words text-xs text-muted-foreground">
Sends two identical requests with a ~9k token system prompt and no cache_control of its own. If LiteLLM
injects the breakpoints, the first call writes the cache and the second reads it. Both calls hit the real
model through the proxy and are billed normally.
</p>
{!enabled && (
<Alert variant="info" className="mt-4">
<CircleAlert />
<AlertTitle>Automatic prompt caching is off</AlertTitle>
<AlertDescription>
Turn it on above to test injection, or run anyway to confirm nothing is injected.
</AlertDescription>
</Alert>
)}
<div className="mt-4 flex flex-wrap items-end gap-4">
<div className="min-w-64">
<ModelSelector
accessToken={accessToken}
value={model}
onChange={setModel}
disabled={running}
labelText="Model"
/>
</div>
<Button onClick={handleRunTest} disabled={running || !model}>
<CirclePlay />
Run test
</Button>
{running && <span className="text-sm text-muted-foreground">Sending call {runningCall} of 2...</span>}
</div>
{error && (
<Alert variant="error" className="mt-4">
<CircleAlert />
<AlertTitle>Test failed</AlertTitle>
<AlertDescription>{error}</AlertDescription>
</Alert>
)}
{outcome && (
<div className="mt-4">
<Table>
<TableHeader>
<TableRow>
<TableHead />
<TableHead>Prompt tokens</TableHead>
<TableHead>Cache write</TableHead>
<TableHead>Cache read</TableHead>
<TableHead>Cost</TableHead>
<TableHead>Latency</TableHead>
</TableRow>
</TableHeader>
<TableBody>
<CallRow label="Call 1" call={outcome.first} />
<CallRow label="Call 2" call={outcome.second} />
</TableBody>
</Table>
{verdict && (
<Alert variant={verdict.variant} className="mt-4">
{outcome.verdict === "injected" || outcome.verdict === "cache_hit_only" ? (
<CircleCheck />
) : (
<CircleAlert />
)}
{verdict.title && <AlertTitle>{verdict.title}</AlertTitle>}
<AlertDescription>{verdict.text(outcome)}</AlertDescription>
</Alert>
)}
</div>
)}
</CardContent>
</Card>
);
};
export default PromptCachingTestCard;

View file

@ -0,0 +1,126 @@
import { describe, expect, it, vi } from "vitest";
import {
PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS,
PromptCachingCallResult,
buildTestSystemPrompt,
judgePromptCachingTest,
runPromptCachingTest,
} from "./promptCachingTest";
const callResult = (overrides: Partial<PromptCachingCallResult> = {}): PromptCachingCallResult => ({
cacheCreationTokens: 0,
cacheReadTokens: 0,
promptTokens: 5000,
responseCost: null,
model: null,
durationMs: 10,
...overrides,
});
const jsonResponse = (body: unknown, init: { status?: number; headers?: Record<string, string> } = {}) =>
new Response(JSON.stringify(body), {
status: init.status ?? 200,
headers: { "Content-Type": "application/json", ...init.headers },
});
const anthropicUsage = {
prompt_tokens: 5010,
cache_creation_input_tokens: 5000,
cache_read_input_tokens: 0,
};
describe("buildTestSystemPrompt", () => {
it("is comfortably above Anthropic's largest minimum cacheable size and differs by nonce", () => {
const first = buildTestSystemPrompt("run-a");
const second = buildTestSystemPrompt("run-b");
expect(first.length).toBeGreaterThan(20000);
expect(first.length).toBeGreaterThan(PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS * 4);
expect(first).not.toBe(second);
expect(first.startsWith("Test run run-a.")).toBe(true);
expect(first.slice(first.indexOf("\n"))).toBe(second.slice(second.indexOf("\n")));
});
});
describe("judgePromptCachingTest", () => {
it("returns injected when call 1 writes and call 2 reads", () => {
expect(
judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult({ cacheReadTokens: 5000 })),
).toBe("injected");
});
it("returns cache_hit_only when neither call writes but a call reads", () => {
expect(judgePromptCachingTest(callResult(), callResult({ cacheReadTokens: 5000 }))).toBe("cache_hit_only");
expect(judgePromptCachingTest(callResult({ cacheReadTokens: 5000 }), callResult({ cacheReadTokens: 5000 }))).toBe(
"cache_hit_only",
);
});
it("returns injected_no_read when call 1 writes but call 2 reads nothing", () => {
expect(judgePromptCachingTest(callResult({ cacheCreationTokens: 5000 }), callResult())).toBe("injected_no_read");
});
it("returns not_injected with no cache activity at all", () => {
expect(judgePromptCachingTest(callResult(), callResult())).toBe("not_injected");
});
});
describe("runPromptCachingTest", () => {
it("sends two sequential identical requests with no cache_control and parses usage", async () => {
const seenBodies: unknown[] = [];
const responses = [
jsonResponse(
{ model: "claude-haiku-4-5", usage: anthropicUsage },
{ headers: { "x-litellm-response-cost": "0.0042" } },
),
jsonResponse({
model: "claude-haiku-4-5",
usage: { prompt_tokens: 5010, prompt_tokens_details: { cached_tokens: 5000 } },
}),
];
const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => {
seenBodies.push(JSON.parse(String(init?.body)));
return responses[seenBodies.length - 1];
});
const testOptions = {
accessToken: "token-123",
model: "claude-haiku-4-5",
baseUrl: "http://proxy:4000",
fetchImpl,
customHeaders: { "x-custom-auth": "abc", Authorization: "Bearer wrong" },
};
const result = await runPromptCachingTest(testOptions);
expect(fetchImpl).toHaveBeenCalledTimes(2);
const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit];
expect(url).toBe("http://proxy:4000/v1/chat/completions");
expect((init.headers as Record<string, string>).Authorization).toBe("Bearer token-123");
expect((init.headers as Record<string, string>)["x-custom-auth"]).toBe("abc");
expect((init.headers as Record<string, string>)["x-litellm-tags"]).toBe("prompt-caching-test");
const serialized = JSON.stringify(seenBodies);
expect(serialized).not.toContain("cache_control");
expect((seenBodies[0] as { messages: { content: string }[] }).messages[0].content).toBe(
(seenBodies[1] as { messages: { content: string }[] }).messages[0].content,
);
expect(result.first.cacheCreationTokens).toBe(5000);
expect(result.first.responseCost).toBeCloseTo(0.0042);
expect(result.second.cacheReadTokens).toBe(5000);
expect(result.verdict).toBe("injected");
});
it("rejects with status and error message on non-2xx", async () => {
const fetchImpl = vi.fn(async () => jsonResponse({ error: { message: "boom" } }, { status: 400 }));
const testOptions = {
accessToken: "token-123",
model: "claude-haiku-4-5",
baseUrl: "http://proxy:4000",
fetchImpl,
};
await expect(runPromptCachingTest(testOptions)).rejects.toThrow(/400.*boom|boom.*400/);
});
});

View file

@ -0,0 +1,178 @@
import { z } from "zod";
import { CustomHeaders, buildPlaygroundHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers";
import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
export const PROMPT_CACHING_TEST_MIN_SYSTEM_TOKENS = 4096;
const FILLER_LINE_COUNT = 600;
export const buildTestSystemPrompt = (nonce: string): string => {
const filler = Array.from(
{ length: FILLER_LINE_COUNT },
(_, index) => `Reference item ${index + 1}: the gateway forwards requests to the configured deployment.`,
).join("\n");
return `Test run ${nonce}.\n${filler}`;
};
export interface PromptCachingTestRequestBody {
readonly model: string;
readonly messages: readonly { readonly role: string; readonly content: string }[];
readonly max_tokens: number;
readonly temperature: number;
}
export const buildTestRequestBody = (model: string, systemPrompt: string): PromptCachingTestRequestBody => ({
model,
messages: [
{ role: "system", content: systemPrompt },
{ role: "user", content: "Reply with the single word OK." },
],
max_tokens: 5,
temperature: 0,
});
export interface PromptCachingCallResult {
cacheCreationTokens: number;
cacheReadTokens: number;
promptTokens: number;
responseCost: number | null;
model: string | null;
durationMs: number;
}
export type PromptCachingVerdict = "injected" | "injected_no_read" | "not_injected" | "cache_hit_only";
export const judgePromptCachingTest = (
first: PromptCachingCallResult,
second: PromptCachingCallResult,
): PromptCachingVerdict => {
if (first.cacheCreationTokens > 0 && second.cacheReadTokens > 0) {
return "injected";
}
if (first.cacheCreationTokens > 0 && second.cacheReadTokens === 0) {
return "injected_no_read";
}
if (first.cacheCreationTokens === 0 && second.cacheCreationTokens === 0) {
if (first.cacheReadTokens > 0 || second.cacheReadTokens > 0) {
return "cache_hit_only";
}
}
return "not_injected";
};
const tokenDetailsSchema = z
.object({
cached_tokens: z.number().nullish(),
cache_write_tokens: z.number().nullish(),
})
.nullish();
const usageFields = {
prompt_tokens: z.number().nullish(),
cache_creation_input_tokens: z.number().nullish(),
cache_read_input_tokens: z.number().nullish(),
prompt_tokens_details: tokenDetailsSchema,
input_tokens_details: tokenDetailsSchema,
};
const usageSchema = z.object(usageFields).nullish();
const chatCompletionResponseSchema = z.object({
model: z.string().nullish(),
usage: usageSchema,
});
const errorBodySchema = z.object({
error: z.object({ message: z.string().optional() }).optional(),
});
const parseResponseCost = (headers: Headers): number | null => {
const raw = headers.get("x-litellm-response-cost");
if (raw === null) {
return null;
}
const cost = Number.parseFloat(raw);
return Number.isFinite(cost) ? cost : null;
};
interface SendTestCallOptions {
readonly accessToken: string;
readonly baseUrl: string;
readonly body: PromptCachingTestRequestBody;
readonly fetchImpl: typeof fetch;
readonly customHeaders: CustomHeaders;
}
const sendTestCall = async ({
accessToken,
baseUrl,
body,
fetchImpl,
customHeaders,
}: SendTestCallOptions): Promise<PromptCachingCallResult> => {
const startedAt = Date.now();
const headers = withRequiredHeaders(buildPlaygroundHeaders(["prompt-caching-test"], customHeaders), {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
});
const response = await fetchImpl(`${baseUrl}/v1/chat/completions`, {
method: "POST",
headers,
body: JSON.stringify(body),
});
const durationMs = Date.now() - startedAt;
const json: unknown = await response.json();
if (!response.ok) {
const parsedError = errorBodySchema.safeParse(json);
const detail = parsedError.success ? parsedError.data.error?.message : undefined;
throw new Error(`Request failed with status ${response.status}${detail ? `: ${detail}` : ""}`);
}
const parsed = chatCompletionResponseSchema.parse(json);
const cacheTokens = extractPromptCacheTokens(parsed.usage);
return {
cacheCreationTokens: cacheTokens.cacheCreationTokens ?? 0,
cacheReadTokens: cacheTokens.cacheReadTokens ?? 0,
promptTokens: parsed.usage?.prompt_tokens ?? 0,
responseCost: parseResponseCost(response.headers),
model: parsed.model ?? null,
durationMs,
};
};
export interface RunPromptCachingTestOptions {
readonly accessToken: string;
readonly model: string;
readonly baseUrl: string;
readonly fetchImpl?: typeof fetch;
readonly onCallStart?: (callIndex: 1 | 2) => void;
readonly customHeaders?: CustomHeaders;
}
export const runPromptCachingTest = async ({
accessToken,
model,
baseUrl,
fetchImpl = fetch,
onCallStart,
customHeaders = {},
}: RunPromptCachingTestOptions): Promise<{
first: PromptCachingCallResult;
second: PromptCachingCallResult;
verdict: PromptCachingVerdict;
}> => {
const systemPrompt = buildTestSystemPrompt(`${Date.now()}-${Math.random().toString(36).slice(2)}`);
const body = buildTestRequestBody(model, systemPrompt);
const callOptions = { accessToken, baseUrl, body, fetchImpl, customHeaders };
onCallStart?.(1);
const first = await sendTestCall(callOptions);
onCallStart?.(2);
const second = await sendTestCall(callOptions);
return { first, second, verdict: judgePromptCachingTest(first, second) };
};

View file

@ -16,11 +16,11 @@ import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"
import RoutingGroups from "@/components/routing_groups";
const PROMPT_CACHING_TAB = "prompt_caching";
const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching";
export const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching";
const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl";
const OPENAI_SYSTEM_MESSAGES_FIRST = "openai_system_messages_first";
const isOn = (value: unknown) => value === true || value === "true";
export const isOn = (value: unknown) => value === true || value === "true";
interface GeneralSettingsPageProps {
accessToken: string | null;