diff --git a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx
index 280e2d10b8c..03d91bb13bc 100644
--- a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx
+++ b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx
@@ -192,9 +192,6 @@ const ChatShell: React.FC<{ children: React.ReactNode }> = ({ children }) => {
))}
-
- Chat is a pre-v0 preview and may change unexpectedly. Do not use in production.
-
diff --git a/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts b/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts
new file mode 100644
index 00000000000..fd3d51883ff
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts
@@ -0,0 +1,177 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { Response as ModelResponse } from "openai/resources/responses/responses";
+import { backgroundResponseUpdate, cancelBackgroundChat, watchBackgroundChat } from "./backgroundChat";
+
+vi.mock("@/components/networking", () => ({ getGatewayBaseUrl: () => "http://localhost:4000/v1" }));
+
+afterEach(() => vi.unstubAllGlobals());
+
+it("should restore text, reasoning, tool results and usage from a completed response", () => {
+ const response = {
+ id: "litellm_poll_example",
+ status: "completed",
+ output: [
+ { type: "reasoning", summary: [{ type: "summary_text", text: "Check the inventory" }] },
+ {
+ type: "mcp_call",
+ id: "call_lookup",
+ name: "lookup",
+ arguments: "{}",
+ output: "12 widgets",
+ server_label: "inventory",
+ },
+ { type: "message", content: [{ type: "output_text", text: "There are 12 widgets." }] },
+ ],
+ usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, output_tokens_details: { reasoning_tokens: 2 } },
+ } as ModelResponse;
+ const update = backgroundResponseUpdate(response);
+ expect(update.content).toBe("There are 12 widgets.");
+ expect(update.reasoningContent).toBe("Check the inventory");
+ expect(update.mcpEvents?.[0].item?.output).toBe("12 widgets");
+ const usage = { promptTokens: 4, completionTokens: 9, totalTokens: 13, reasoningTokens: 2 };
+ expect(update.usage).toMatchObject(usage);
+ expect(update.backgroundRun).toEqual({ id: response.id, status: "completed" });
+});
+
+describe("background response lifecycle", () => {
+ it("should detach polling without cancelling the server run", async () => {
+ const controller = new AbortController();
+ const fetch = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) =>
+ Response.json({ id: "litellm_poll_example", status: "in_progress", output: [] }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ const onUpdate = vi.fn(() => controller.abort());
+ const polling = {
+ accessToken: "test-token",
+ responseId: "litellm_poll_example",
+ signal: controller.signal,
+ onUpdate,
+ };
+ await watchBackgroundChat(polling);
+ expect(onUpdate).toHaveBeenCalledOnce();
+ expect(fetch).toHaveBeenCalledOnce();
+ expect(fetch.mock.calls[0][1]?.method).toBe("GET");
+ });
+
+ it("should mark an expired run unavailable without clearing its partial answer", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => Response.json({ error: { message: "Expired" } }, { status: 404 })),
+ );
+ const onUpdate = vi.fn();
+ const polling = {
+ accessToken: "test-token",
+ responseId: "litellm_poll_expired",
+ signal: new AbortController().signal,
+ onUpdate,
+ };
+ await watchBackgroundChat(polling);
+ expect(onUpdate).toHaveBeenCalledWith({
+ backgroundRun: { id: "litellm_poll_expired", status: "expired" },
+ backgroundError: "This saved run is no longer available. Its result may have expired on the server.",
+ });
+ });
+
+ it("should cancel explicitly through the server and preserve returned partial output", async () => {
+ const fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+ const request = input instanceof Request ? input : new Request(input, init);
+ expect(request.method).toBe("POST");
+ expect(new URL(request.url).pathname).toBe("/v1/responses/litellm_poll_example/cancel");
+ return Response.json({
+ id: "litellm_poll_example",
+ status: "cancelled",
+ output: [{ type: "message", content: [{ type: "output_text", text: "Partial answer" }] }],
+ });
+ });
+ vi.stubGlobal("fetch", fetch);
+ expect(await cancelBackgroundChat("test-token", "litellm_poll_example")).toMatchObject({
+ content: "Partial answer",
+ backgroundRun: { id: "litellm_poll_example", status: "cancelled" },
+ });
+ });
+});
+
+it.each([
+ [0.125, 0.125],
+ ["0.125", 0.125],
+ [0, 0],
+ ["invalid", undefined],
+])("should preserve valid returned cost %s", (cost, expected) => {
+ const response = {
+ id: "litellm_poll_cost",
+ status: "completed",
+ output: [],
+ usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost },
+ } as ModelResponse;
+ expect(backgroundResponseUpdate(response).usage?.cost).toBe(expected);
+});
+
+it("should keep polling through unfinished output items until the answer and tools arrive", async () => {
+ let polls = 0;
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => {
+ polls++;
+ return Response.json({
+ id: "litellm_poll_partial",
+ status: polls === 1 ? "in_progress" : "completed",
+ output:
+ polls === 1
+ ? [
+ { type: "message", id: "message_1" },
+ { type: "mcp_list_tools", id: "tools_1", server_label: "inventory" },
+ ]
+ : [
+ { type: "message", content: [{ type: "output_text", text: "12 widgets" }] },
+ { type: "mcp_list_tools", server_label: "inventory", tools: [{ name: "lookup", input_schema: {} }] },
+ ],
+ });
+ }),
+ );
+ const onUpdate = vi.fn();
+ const polling = {
+ accessToken: "test-token",
+ responseId: "litellm_poll_partial",
+ signal: new AbortController().signal,
+ onUpdate,
+ };
+ await watchBackgroundChat(polling);
+ expect(polls).toBe(2);
+ expect(onUpdate.mock.lastCall?.[0]).toMatchObject({
+ content: "12 widgets",
+ backgroundRun: { status: "completed" },
+ mcpEvents: [{ item: { tools: [{ name: "lookup" }] } }],
+ });
+});
+
+it("should restore server timings from polling without counting time away from the chat", async () => {
+ const response = {
+ id: "litellm_poll_timing",
+ status: "completed",
+ created_at: 1000,
+ output: [],
+ litellm_response_timing: { time_to_first_token_ms: 250, total_latency_ms: 12500 },
+ };
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => Response.json(response)),
+ );
+ const onUpdate = vi.fn();
+ const polling = {
+ accessToken: "test-token",
+ responseId: "litellm_poll_timing",
+ signal: new AbortController().signal,
+ onUpdate,
+ };
+ await watchBackgroundChat(polling);
+ expect(onUpdate).toHaveBeenCalledWith(expect.objectContaining({ timeToFirstToken: 250, totalLatency: 12500 }));
+});
+
+it.each(["queued", "completed"] as const)(
+ "should not invent timings for a %s response without measurements",
+ (status) => {
+ const update = backgroundResponseUpdate({ id: "litellm_poll_legacy", status, output: [] } as ModelResponse);
+ expect(update).not.toHaveProperty("timeToFirstToken");
+ expect(update).not.toHaveProperty("totalLatency");
+ },
+);
diff --git a/ui/litellm-dashboard/src/components/chat/backgroundChat.ts b/ui/litellm-dashboard/src/components/chat/backgroundChat.ts
new file mode 100644
index 00000000000..177d25f2e16
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/chat/backgroundChat.ts
@@ -0,0 +1,138 @@
+import OpenAI from "openai";
+import type { Response } from "openai/resources/responses/responses";
+import { getGatewayBaseUrl } from "@/components/networking";
+import { parseUsageCost } from "@/components/llm_calls/usage_cost";
+import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
+import type { AssistantMessageUpdate, ChatBackgroundRun, MCPEvent } from "./types";
+
+export const isBackgroundRunPending = (run?: ChatBackgroundRun): boolean =>
+ run?.status === "queued" || run?.status === "in_progress";
+
+type TimedResponse = Response & {
+ litellm_response_timing?: {
+ time_to_first_token_ms?: number;
+ total_latency_ms?: number;
+ };
+};
+
+export function backgroundResponseUpdate(response: TimedResponse): AssistantMessageUpdate {
+ const output = response.output ?? [];
+ const content = output
+ .filter((item) => item.type === "message")
+ .flatMap((item) => item.content ?? [])
+ .map((part) => (part.type === "output_text" ? part.text : part.refusal))
+ .join("");
+ const reasoningContent = output
+ .filter((item) => item.type === "reasoning")
+ .flatMap((item) => item.summary ?? [])
+ .map((part) => part.text)
+ .join("\n\n");
+ const mcpEvents: MCPEvent[] = output.flatMap
((item) => {
+ if (item.type === "mcp_call") {
+ return [{ type: "response.output_item.done", item: { ...item, output: item.output ?? undefined } }];
+ }
+ if (item.type === "mcp_list_tools") {
+ return [
+ {
+ type: "response.output_item.done",
+ item: {
+ ...item,
+ tools: (item.tools ?? []).map((tool) => ({
+ name: tool.name,
+ input_schema: tool.input_schema,
+ description: tool.description ?? "",
+ })),
+ },
+ },
+ ];
+ }
+ return [];
+ });
+ const status = response.status ?? "in_progress";
+ const errors: Partial> = {
+ failed: response.error?.message || "The background run failed.",
+ incomplete: "The run ended before completing its answer.",
+ };
+ const backgroundError = errors[status];
+ const cost = response.usage && "cost" in response.usage ? parseUsageCost(response.usage.cost) : undefined;
+ const timing = response.litellm_response_timing;
+ return {
+ content,
+ reasoningContent,
+ mcpEvents,
+ backgroundRun: { id: response.id, status },
+ backgroundError,
+ ...(timing?.time_to_first_token_ms !== undefined && { timeToFirstToken: timing.time_to_first_token_ms }),
+ ...(timing?.total_latency_ms !== undefined && { totalLatency: timing.total_latency_ms }),
+ ...(response.usage && {
+ usage: {
+ promptTokens: response.usage.input_tokens,
+ completionTokens: response.usage.output_tokens,
+ totalTokens: response.usage.total_tokens,
+ reasoningTokens: response.usage.output_tokens_details?.reasoning_tokens,
+ ...extractPromptCacheTokens(response.usage),
+ ...(cost !== undefined ? { cost } : {}),
+ },
+ }),
+ };
+}
+
+const backgroundClient = (accessToken: string) =>
+ new OpenAI({
+ apiKey: accessToken,
+ baseURL: getGatewayBaseUrl(),
+ dangerouslyAllowBrowser: true,
+ });
+
+function waitForPoll(signal: AbortSignal): Promise {
+ return new Promise((resolve, reject) => {
+ const abort = () => {
+ clearTimeout(timer);
+ reject(signal.reason);
+ };
+ const timer = setTimeout(() => {
+ signal.removeEventListener("abort", abort);
+ resolve();
+ }, 1000);
+ signal.addEventListener("abort", abort, { once: true });
+ if (signal.aborted) abort();
+ });
+}
+
+export async function watchBackgroundChat({
+ accessToken,
+ responseId,
+ signal,
+ onUpdate,
+}: {
+ accessToken: string;
+ responseId: string;
+ signal: AbortSignal;
+ onUpdate: (update: AssistantMessageUpdate) => void;
+}): Promise {
+ const client = backgroundClient(accessToken);
+ while (!signal.aborted) {
+ try {
+ const response = await client.responses.retrieve(responseId, { stream: false }, { signal });
+ if (signal.aborted) return;
+ onUpdate(backgroundResponseUpdate(response));
+ if (!isBackgroundRunPending({ id: response.id, status: response.status ?? "in_progress" })) return;
+ await waitForPoll(signal);
+ } catch (error) {
+ if (signal.aborted) return;
+ if (error instanceof OpenAI.APIError && error.status === 404) {
+ onUpdate({
+ backgroundRun: { id: responseId, status: "expired" },
+ backgroundError: "This saved run is no longer available. Its result may have expired on the server.",
+ });
+ return;
+ }
+ throw error;
+ }
+ }
+}
+
+export async function cancelBackgroundChat(accessToken: string, responseId: string): Promise {
+ const response = await backgroundClient(accessToken).responses.cancel(responseId);
+ return backgroundResponseUpdate(response);
+}
diff --git a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts
index 157363d2461..9e8016b608e 100644
--- a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts
+++ b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts
@@ -1,5 +1,6 @@
import OpenAI from "openai";
import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api";
+import { backgroundResponseUpdate } from "./backgroundChat";
import { generateChatImages } from "./imageHelpers";
import { toResponsesMessages } from "./attachmentHelpers";
import type { AssistantMessageUpdate, ChatMessage, MCPEvent } from "./types";
@@ -15,6 +16,7 @@ interface ChatRequestOptions {
"role" | "content" | "attachments" | "generatedImages" | "imageGeneration" | "mcpEvents"
>[];
previousResponseId?: string | null;
+ background?: boolean;
mcpServers?: string[];
onResponseId: (id: string | null) => void;
onUpdate: (update: AssistantMessageUpdate) => void;
@@ -43,6 +45,7 @@ export async function runChatRequest({
signal,
messages,
previousResponseId,
+ background = false,
mcpServers,
onResponseId,
onUpdate,
@@ -94,6 +97,14 @@ export async function runChatRequest({
undefined,
true,
(totalLatency) => onUpdate({ totalLatency }),
+ undefined,
+ undefined,
+ background
+ ? (response) => {
+ onResponseId(null);
+ onUpdate(backgroundResponseUpdate(response));
+ }
+ : undefined,
);
signal.throwIfAborted();
} catch (error: unknown) {
diff --git a/ui/litellm-dashboard/src/components/chat/types.ts b/ui/litellm-dashboard/src/components/chat/types.ts
index 25b232e40c4..e98b61e2fe4 100644
--- a/ui/litellm-dashboard/src/components/chat/types.ts
+++ b/ui/litellm-dashboard/src/components/chat/types.ts
@@ -25,6 +25,11 @@ export interface ChatImageGeneration {
prompt: string;
}
+export interface ChatBackgroundRun {
+ id: string;
+ status: "queued" | "in_progress" | "completed" | "failed" | "cancelled" | "incomplete" | "expired";
+}
+
export interface ChatMessage {
id: string;
role: "user" | "assistant" | "tool";
@@ -33,6 +38,8 @@ export interface ChatMessage {
generatedImages?: ChatImageAttachment[];
imageGeneration?: ChatImageGeneration;
reasoningContent?: string;
+ backgroundRun?: ChatBackgroundRun;
+ backgroundError?: string;
/** MCP tool events that occurred during this assistant turn, in order. */
mcpEvents?: MCPEvent[];
toolName?: string;
@@ -55,6 +62,8 @@ export type AssistantMessageUpdate = Partial<
| "usage"
| "generatedImages"
| "imageGeneration"
+ | "backgroundRun"
+ | "backgroundError"
>
>;
diff --git a/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts b/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts
new file mode 100644
index 00000000000..c0d1c43bcde
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts
@@ -0,0 +1,62 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { cancelBackgroundChat, isBackgroundRunPending, watchBackgroundChat } from "./backgroundChat";
+import type { AssistantMessageUpdate, ChatBackgroundRun } from "./types";
+
+export function useBackgroundChat(
+ conversationId: string | null,
+ run: ChatBackgroundRun | undefined,
+ accessToken: string,
+ updateMessage: (conversationId: string, update: AssistantMessageUpdate) => void,
+) {
+ const [failure, setFailure] = useState<{ responseId: string; retry: number; message: string } | null>(null);
+ const [retry, setRetry] = useState(0);
+ const [cancellingId, setCancellingId] = useState(null);
+ const controllerRef = useRef(null);
+ const pending = isBackgroundRunPending(run);
+ const responseId = run?.id;
+ const cancelling = Boolean(responseId && cancellingId === responseId);
+ const connectionError =
+ pending && failure?.responseId === responseId && failure?.retry === retry ? failure.message : null;
+
+ useEffect(() => {
+ if (!conversationId || !responseId || !pending) return;
+ const controller = new AbortController();
+ controllerRef.current = controller;
+ const polling = {
+ accessToken,
+ responseId,
+ signal: controller.signal,
+ onUpdate: (update: AssistantMessageUpdate) => updateMessage(conversationId, update),
+ };
+ void watchBackgroundChat(polling).catch(() => {
+ if (!controller.signal.aborted)
+ setFailure({ responseId, retry, message: "Connection lost. Your run may still be running on the server." });
+ });
+ return () => {
+ controller.abort();
+ if (controllerRef.current === controller) controllerRef.current = null;
+ };
+ }, [conversationId, responseId, pending, accessToken, updateMessage, retry]);
+
+ const reconnect = useCallback(() => setRetry((value) => value + 1), []);
+
+ const cancel = useCallback(async () => {
+ if (!conversationId || !responseId || cancelling) return;
+ const controller = controllerRef.current;
+ controller?.abort();
+ setCancellingId(responseId);
+ try {
+ const update = await cancelBackgroundChat(accessToken, responseId);
+ updateMessage(conversationId, update);
+ if (controllerRef.current === controller) reconnect();
+ } catch {
+ if (controllerRef.current === controller) {
+ setFailure({ responseId, retry, message: "Could not stop the run. Reconnect and try Stop again." });
+ }
+ } finally {
+ setCancellingId(null);
+ }
+ }, [conversationId, responseId, accessToken, cancelling, reconnect, retry, updateMessage]);
+
+ return { pending, connectionError, cancelling, reconnect, cancel };
+}
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
index d38db631c6e..cac91cf92d4 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
@@ -9,6 +9,7 @@ export interface ModelGroup {
mode?: string;
supports_reasoning?: boolean;
supports_fast_mode?: boolean;
+ supports_background_chat?: boolean;
supported_reasoning_efforts?: string[] | null;
}
@@ -20,6 +21,7 @@ interface AvailableModel {
mode?: string | null;
supports_reasoning?: boolean | null;
supports_fast_mode?: boolean | null;
+ supports_background_chat?: boolean | null;
supported_reasoning_efforts?: string[] | null;
}
@@ -30,6 +32,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => {
...(item.providers && { providers: item.providers }),
...(item.mode && { mode: item.mode }),
...(item.supports_reasoning === true && { supports_reasoning: true }),
+ ...(item.supports_background_chat === true && { supports_background_chat: true }),
...(item.supports_fast_mode === true && { supports_fast_mode: true }),
...(item.supported_reasoning_efforts !== undefined && {
supported_reasoning_efforts: item.supported_reasoning_efforts,
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
index ce1a8aab0c5..ef737c22cdb 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
@@ -1,6 +1,10 @@
import { observeResponseHeaders, type ResponseHeadersListener } from "@/lib/http/observeResponseHeaders";
import openai from "openai";
-import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses";
+import type {
+ Response as OpenAIResponse,
+ ResponseFunctionToolCall,
+ ResponseInputItem,
+} from "openai/resources/responses/responses";
import { MessageType } from "../chat_ui/types";
import { TokenUsage } from "../chat_ui/ResponseMetrics";
import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking";
@@ -110,6 +114,7 @@ export async function makeOpenAIResponsesRequest(
onTotalLatency?: (latency: number) => void,
customHeaders?: CustomHeaders,
onResponseHeaders?: ResponseHeadersListener,
+ onBackgroundResponse?: (response: OpenAIResponse) => void,
) {
if (!accessToken) {
throw new Error("Virtual Key is required");
@@ -239,6 +244,16 @@ export async function makeOpenAIResponsesRequest(
...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
};
+ if (onBackgroundResponse) {
+ const response = await client.post("/responses", {
+ body: { ...requestBody, background: true, stream: false },
+ signal,
+ maxRetries: 0,
+ });
+ onBackgroundResponse(response);
+ return response;
+ }
+
// Create request to OpenAI responses API
// Use 'any' type to avoid TypeScript issues with the experimental API
const response = streamingEnabled
diff --git a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx
index f93e01eb583..32e590017b0 100644
--- a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx
@@ -19,7 +19,6 @@ vi.mock("../../utils/dataUtils", () => ({
}));
vi.mock("./networking", () => ({
- modelInfoV1Call: vi.fn(),
credentialGetCall: vi.fn(),
credentialListCall: vi.fn(),
getGuardrailsList: vi.fn(),
@@ -56,7 +55,6 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () =>
}));
const mockToast = vi.mocked(toast);
-const mockModelInfoV1Call = vi.mocked(networking.modelInfoV1Call);
const mockCredentialGetCall = vi.mocked(networking.credentialGetCall);
const mockCredentialListCall = vi.mocked(networking.credentialListCall);
const mockGetGuardrailsList = vi.mocked(networking.getGuardrailsList);
@@ -134,10 +132,6 @@ describe("ModelInfoView", () => {
error: null,
});
- mockModelInfoV1Call.mockResolvedValue({
- data: [defaultModelData],
- });
-
mockCredentialGetCall.mockResolvedValue({
credential_name: "test-credential",
credential_values: {},
@@ -394,7 +388,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, mode: "chat", key: "anthropic.claude-haiku-4-5" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [displayedModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [displayedModel] });
render(, { wrapper });
await waitFor(() => {
@@ -869,7 +862,6 @@ describe("ModelInfoView", () => {
const renderWithPtuModel = () => {
mockUseModelsInfo.mockReturnValue({ data: { data: [ptuModelData] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [ptuModelData] });
return render(, { wrapper });
};
@@ -967,7 +959,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1", input_cost_per_token: 0.0000003 },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [seededModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [seededModel] });
const user = userEvent.setup();
render(, { wrapper });
@@ -1008,7 +999,6 @@ describe("ModelInfoView", () => {
model_info: { ...ptuModelData.model_info, cost_per_ptu_per_hour: null, ptu_effective_from: null },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [halfSetPtuModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [halfSetPtuModel] });
const user = userEvent.setup();
render(, { wrapper });
@@ -1057,7 +1047,6 @@ describe("ModelInfoView", () => {
model_info: { ...ptuModelData.model_info, ptu_effective_from: null, ptu_effective_to: null },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [undatedPtuModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [undatedPtuModel] });
const user = userEvent.setup();
render(, { wrapper });
@@ -1201,7 +1190,6 @@ describe("ModelInfoView", () => {
isLoading: false,
error: null,
});
- mockModelInfoV1Call.mockResolvedValue({ data: [maskedModelData] });
const user = userEvent.setup();
render(, { wrapper });
@@ -1508,7 +1496,6 @@ describe("ModelInfoView", () => {
beforeEach(() => {
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
});
const readRawJson = async (user: ReturnType) => {
@@ -1555,7 +1542,6 @@ describe("ModelInfoView", () => {
it("shows Not Set and no team_alias for a model without a team", async () => {
mockUseModelsInfo.mockReturnValue({ data: { data: [defaultModelData] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [defaultModelData] });
mockUseTeams.mockReturnValue({
data: [{ team_id: "team-1", team_alias: "alpha" }],
isLoading: false,
@@ -1849,7 +1835,6 @@ describe("ModelInfoView", () => {
beforeEach(() => {
mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheWriteTiers] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [withCacheWriteTiers] });
});
it("shows both configured cache write rates per million tokens", async () => {
@@ -1926,7 +1911,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
const user = userEvent.setup();
await enterEditMode(user);
@@ -1952,7 +1936,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
const user = userEvent.setup();
await enterEditMode(user);
@@ -1992,7 +1975,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
const user = userEvent.setup();
render(, { wrapper });
await user.click(await screen.findByRole("button", { name: /edit settings/i }));
@@ -2069,7 +2051,6 @@ describe("ModelInfoView", () => {
litellm_params: litellmParamsWithoutCredential,
};
mockUseModelsInfo.mockReturnValue({ data: { data: [modelWithoutCredential] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [modelWithoutCredential] });
const user = userEvent.setup();
await enterEditMode(user);
@@ -2150,7 +2131,6 @@ describe("ModelInfoView", () => {
litellm_params: { ...defaultModelData.litellm_params, model: "openai/gpt-4*" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [wildcard] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [wildcard] });
mockUseModelHub.mockReturnValue({
data: { data: [{ model_group: "openai/gpt-4o", providers: ["openai"] }] },
isLoading: false,
@@ -2207,7 +2187,6 @@ describe("ModelInfoView", () => {
litellm_params: { ...defaultModelData.litellm_params, cache_control_injection_points: points },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [data] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [data] });
};
it("omits the key when the deployment has none and the operator leaves the toggle alone", async () => {
@@ -2330,7 +2309,6 @@ describe("ModelInfoView", () => {
litellm_params: { ...defaultModelData.litellm_params, cache_read_input_token_cost: 0.000001 },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheRead] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [withCacheRead] });
const user = userEvent.setup();
await enterEditMode(user);
setInputCost("2");
@@ -2365,7 +2343,6 @@ describe("ModelInfoView", () => {
},
};
mockUseModelsInfo.mockReturnValue({ data: { data: [resolved] }, isLoading: false, error: null });
- mockModelInfoV1Call.mockResolvedValue({ data: [resolved] });
const user = userEvent.setup();
await enterEditMode(user);
setInputCost("5");
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index c13052be245..994c2948346 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -48,7 +48,7 @@ export const getAutoRouterClassifierDefaultPromptCall = async (
}
};
-export type AssembledPromptTierSource =
+type AssembledPromptTierSource =
| { tierDefinitions: { name: string; description?: string }[] }
| { tierLabels?: Record; classificationRubric?: string };
@@ -127,16 +127,6 @@ import type {
SubscriptionUsageResponse,
} from "@/app/(dashboard)/usage/_components/components/SubscriptionCost/types";
import type { RoutingDecision } from "./logs/detail/RoutingDecisionCard";
-import type {
- SpanDetail,
- SpanErrorPage,
- SpanErrorQuery,
- SpanQuery,
- Trace,
- TraceDetailQuery,
- TraceListQuery,
- TracePage,
-} from "@litellm/lens-ui";
import {
createApiClient,
deriveErrorMessage,
@@ -541,7 +531,7 @@ export const apiClient = createApiClient({
onError: handleError,
});
-export const gatewayApiClient = createApiClient({
+const gatewayApiClient = createApiClient({
getBaseUrl: getGatewayBaseUrl,
getAuthHeaderName: getGlobalLitellmHeaderName,
onError: handleError,
@@ -1725,37 +1715,6 @@ export const modelInfoCall = async (
}
};
-export const modelInfoV1Call = async (accessToken: string, modelId: string) => {
- /**
- * Get all models on proxy
- */
- try {
- let url = proxyBaseUrl ? `${proxyBaseUrl}/v1/model/info` : `/v1/model/info`;
- url += `?litellm_model_id=${modelId}`;
-
- const response = await fetch(url, {
- method: "GET",
- headers: {
- [globalLitellmHeaderName]: `Bearer ${accessToken}`,
- "Content-Type": "application/json",
- },
- });
-
- if (!response.ok) {
- const errorData = await response.json();
- const errorMessage = deriveErrorMessage(errorData);
- handleError(errorMessage);
- throw new Error(errorMessage);
- }
-
- const data = await response.json();
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
export const modelHubPublicModelsCall = async () => {
const url = proxyBaseUrl ? `${proxyBaseUrl}/public/model_hub` : `/public/model_hub`;
const response = await fetch(url, {
@@ -1906,66 +1865,6 @@ export const modelAvailableCall = async (
}
};
-export const teamSpendLogsCall = async (accessToken: string) => {
- try {
- const data = await apiClient.get(`/global/spend/teams`, { accessToken });
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
-export const tagsSpendLogsCall = async (
- accessToken: string,
- startTime: string | undefined,
- endTime: string | undefined,
- tags: string[] | undefined,
-) => {
- try {
- let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/tags` : `/global/spend/tags`;
-
- if (startTime && endTime) {
- url = `${url}?start_date=${startTime}&end_date=${endTime}`;
- }
-
- // if tags, convert the list to a comma separated string
- if (tags) {
- url += `&tags=${tags.join(",")}`;
- }
-
- const response = await fetch(`${url}`, {
- method: "GET",
- headers: {
- [globalLitellmHeaderName]: `Bearer ${accessToken}`,
- "Content-Type": "application/json",
- },
- });
- if (!response.ok) {
- const errorData = await response.json();
- const errorMessage = deriveErrorMessage(errorData);
- handleError(errorMessage);
- throw new Error(errorMessage);
- }
-
- const data = await response.json();
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
-export const allTagNamesCall = async (accessToken: string) => {
- try {
- const data = await apiClient.get(`/global/spend/all_tag_names`, { accessToken });
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
export const userFilterUICall = async (accessToken: string, params: URLSearchParams) => {
try {
return await apiClient.get(`/user/filter/ui`, {
@@ -2081,214 +1980,6 @@ export const uiSpendLogsCall = async ({
}
};
-/**
- * Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not
- * configured; callers can detect that through the thrown `ApiError`'s `status`.
- */
-export const agentTraceListCall = async ({
- accessToken,
- startMs,
- endMs,
- cursor,
-}: {
- accessToken: string;
- startMs: number;
- endMs: number;
- cursor?: string | null;
-}): Promise => {
- const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined } satisfies TraceListQuery;
- return apiClient.get(`/v1/traces`, { accessToken, query });
-};
-
-export const sendOtlpTraceCall = async (accessToken: string, exportRequest: object): Promise =>
- apiClient.post(`/v1/traces`, { accessToken, body: exportRequest });
-
-export const agentTraceCall = async (
- accessToken: string,
- traceId: string,
- traceRef?: string,
- cursor?: string | null,
-): Promise =>
- apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, {
- accessToken,
- query: { trace_ref: traceRef || undefined, cursor: cursor ?? undefined, page_size: 200 } satisfies TraceDetailQuery,
- });
-
-export const agentTraceSpanCall = async (
- accessToken: string,
- traceId: string,
- spanId: string,
- traceRef?: string,
-): Promise =>
- apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, {
- accessToken,
- query: { trace_ref: traceRef || undefined } satisfies SpanQuery,
- });
-
-export const agentTraceSpanErrorCall = async (
- accessToken: string,
- traceId: string,
- spanId: string,
- options: { traceRef?: string; cursor?: string | null },
-): Promise =>
- apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}/error`, {
- accessToken,
- query: { trace_ref: options.traceRef || undefined, cursor: options.cursor || undefined } satisfies SpanErrorQuery,
- });
-
-export const adminSpendLogsCall = async (accessToken: string) => {
- try {
- const data = await apiClient.get(`/global/spend/logs`, { accessToken });
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
-export const adminTopKeysCall = async (accessToken: string) => {
- try {
- let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/keys?limit=5` : `/global/spend/keys?limit=5`;
-
- const response = await fetch(url, {
- method: "GET",
- headers: {
- [globalLitellmHeaderName]: `Bearer ${accessToken}`,
- "Content-Type": "application/json",
- },
- });
- if (!response.ok) {
- const errorData = await response.json();
- const errorMessage = deriveErrorMessage(errorData);
- handleError(errorMessage);
- throw new Error(errorMessage);
- }
-
- const data = await response.json();
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
-export const adminTopEndUsersCall = async (
- accessToken: string,
- keyToken: string | null,
- startTime: string | undefined,
- endTime: string | undefined,
-) => {
- try {
- const body = keyToken
- ? { api_key: keyToken, startTime: startTime, endTime: endTime }
- : { startTime: startTime, endTime: endTime };
-
- const data = await apiClient.post(`/global/spend/end_users`, { accessToken, body });
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
-export const adminspendByProvider = async (
- accessToken: string,
- startTime: string | undefined,
- endTime: string | undefined,
-) => {
- try {
- const data = await apiClient.get(`/global/spend/provider`, {
- accessToken,
- query: {
- ...(startTime && endTime ? { start_date: startTime, end_date: endTime } : {}),
- },
- });
- return data;
- } catch (error) {
- console.error("Failed to fetch spend data:", error);
- throw error;
- }
-};
-
-export const adminGlobalActivity = async (
- accessToken: string,
- startTime: string | undefined,
- endTime: string | undefined,
-) => {
- try {
- const data = await apiClient.get(`/global/activity`, {
- accessToken,
- query: startTime && endTime ? { start_date: startTime, end_date: endTime } : undefined,
- });
- return data;
- } catch (error) {
- console.error("Failed to fetch spend data:", error);
- throw error;
- }
-};
-
-export const adminGlobalActivityPerModel = async (
- accessToken: string,
- startTime: string | undefined,
- endTime: string | undefined,
-) => {
- try {
- let url = proxyBaseUrl ? `${proxyBaseUrl}/global/activity/model` : `/global/activity/model`;
-
- if (startTime && endTime) {
- url += `?start_date=${startTime}&end_date=${endTime}`;
- }
-
- const requestOptions = {
- method: "GET",
- headers: {
- [globalLitellmHeaderName]: `Bearer ${accessToken}`,
- },
- };
-
- const response = await fetch(url, requestOptions);
-
- if (!response.ok) {
- const errorData = await response.json();
- const errorMessage = deriveErrorMessage(errorData);
- handleError(errorMessage);
- throw new Error(errorMessage);
- }
-
- const data = await response.json();
- return data;
- } catch (error) {
- console.error("Failed to fetch spend data:", error);
- throw error;
- }
-};
-
-export const adminTopModelsCall = async (accessToken: string) => {
- try {
- let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/models?limit=5` : `/global/spend/models?limit=5`;
-
- const response = await fetch(url, {
- method: "GET",
- headers: {
- [globalLitellmHeaderName]: `Bearer ${accessToken}`,
- "Content-Type": "application/json",
- },
- });
- if (!response.ok) {
- const errorData = await response.json();
- const errorMessage = deriveErrorMessage(errorData);
- handleError(errorMessage);
- throw new Error(errorMessage);
- }
-
- const data = await response.json();
- return data;
- } catch (error) {
- console.error("Failed to create key:", error);
- throw error;
- }
-};
-
export const testConnectionRequest = async (
accessToken: string,
litellm_params: Record,
@@ -2421,7 +2112,7 @@ export const testAutoRouterRouting = async (
}
};
-export interface ComplexityRouterConfigValidation {
+interface ComplexityRouterConfigValidation {
valid: boolean;
error?: string | null;
}
@@ -2733,7 +2424,7 @@ export interface UserConnectionStartResponse {
flow_handle: string;
}
-export type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected";
+type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected";
export interface UserConnectionPollResponse {
status: UserConnectionPollStatus;
@@ -3697,35 +3388,6 @@ export const getProxyUISettings = async (accessToken: string) => {
}
};
-export const getUISettings = async (accessToken: string) => {
- /**
- * Get UI-specific configuration flags from the database
- */
- try {
- const url = proxyBaseUrl ? `${proxyBaseUrl}/get/ui_settings` : `/get/ui_settings`;
- const response = await fetch(url, {
- method: "GET",
- headers: {
- [globalLitellmHeaderName]: `Bearer ${accessToken}`,
- "Content-Type": "application/json",
- },
- });
-
- if (!response.ok) {
- const errorData = await response.json();
- const errorMessage = deriveErrorMessage(errorData);
- console.error("Failed to get UI settings:", errorMessage);
- return null;
- }
-
- const data = await response.json();
- return data;
- } catch (error) {
- console.error("Failed to get UI settings:", error);
- return null;
- }
-};
-
export const getMCPSemanticFilterSettings = async (accessToken: string) => {
/**
* Get MCP semantic filter configuration
@@ -4685,12 +4347,12 @@ export const createAgentCall = async (accessToken: string, agentData: any) => {
export type DiscoveredAgentCard = Agent["agent_card_params"];
-export type DiscoverAgentCardResponse = Omit & {
+type DiscoverAgentCardResponse = Omit & {
agent_card: DiscoveredAgentCard;
};
export type DiscoveryMode = components["schemas"]["DiscoveryMode"];
-export type DiscoverAgentCardOptions = Partial<
+type DiscoverAgentCardOptions = Partial<
Pick
>;
@@ -5613,7 +5275,7 @@ export const vectorStoreListCall = async (
}
};
-export interface IndexesListResponse {
+interface IndexesListResponse {
object: string;
data: VectorStoreIndex[];
}
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index e80df24a818..0dce821f862 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -5849,6 +5849,26 @@ export interface paths {
patch?: never;
trace?: never;
};
+ "/get/background_chat_settings": {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ /**
+ * Get Background Chat Settings
+ * @description Get saved background-response settings and the prerequisites for changing them.
+ */
+ get: operations["get_background_chat_settings_get_background_chat_settings_get"];
+ put?: never;
+ post?: never;
+ delete?: never;
+ options?: never;
+ head?: never;
+ patch?: never;
+ trace?: never;
+ };
"/get/config/callbacks": {
parameters: {
query?: never;
@@ -18240,6 +18260,26 @@ export interface paths {
patch: operations["typesafe_proxy_route_typesafe__endpoint__patch"];
trace?: never;
};
+ "/update/background_chat_settings": {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ get?: never;
+ put?: never;
+ post?: never;
+ delete?: never;
+ options?: never;
+ head?: never;
+ /**
+ * Update Background Chat Settings
+ * @description Save background-response polling settings for the proxy configuration refresh.
+ */
+ patch: operations["update_background_chat_settings_update_background_chat_settings_patch"];
+ trace?: never;
+ };
"/update/default_team_settings": {
parameters: {
query?: never;
@@ -27157,6 +27197,37 @@ export interface components {
} & {
[key: string]: unknown;
};
+ /** BackgroundChatSettings */
+ BackgroundChatSettings: {
+ /**
+ * Enabled
+ * @description Enable cache-backed background Responses for Chat and API requests.
+ */
+ enabled: boolean;
+ };
+ /** BackgroundChatSettingsResponse */
+ BackgroundChatSettingsResponse: {
+ /**
+ * Can Save
+ * @description Whether database-backed configuration updates are available.
+ */
+ can_save: boolean;
+ /**
+ * Enabled
+ * @description Enable cache-backed background Responses for Chat and API requests.
+ */
+ enabled: boolean;
+ /**
+ * Managed By Config
+ * @description Whether the responses configuration is owned by the config file.
+ */
+ managed_by_config: boolean;
+ /**
+ * Redis Configured
+ * @description Whether this proxy has coordination Redis configured.
+ */
+ redis_configured: boolean;
+ };
/** BaseLitellmParams */
BaseLitellmParams: {
/**
@@ -38804,6 +38875,12 @@ export interface components {
supported_openai_params: string[] | null;
/** Supported Reasoning Efforts */
supported_reasoning_efforts?: string[] | null;
+ /**
+ * Supports Background Chat
+ * @description Whether cache-backed background Responses runs are enabled for this model group
+ * @default false
+ */
+ supports_background_chat: boolean;
/**
* Supports Fast Mode
* @default false
@@ -60982,6 +61059,26 @@ export interface operations {
};
};
};
+ get_background_chat_settings_get_background_chat_settings_get: {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ requestBody?: never;
+ responses: {
+ /** @description Successful Response */
+ 200: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["BackgroundChatSettingsResponse"];
+ };
+ };
+ };
+ };
get_config_get_config_callbacks_get: {
parameters: {
query?: never;
@@ -76302,6 +76399,39 @@ export interface operations {
};
};
};
+ update_background_chat_settings_update_background_chat_settings_patch: {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ requestBody: {
+ content: {
+ "application/json": components["schemas"]["BackgroundChatSettings"];
+ };
+ };
+ responses: {
+ /** @description Successful Response */
+ 200: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["BackgroundChatSettingsResponse"];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
update_default_team_settings_update_default_team_settings_patch: {
parameters: {
query?: never;