diff --git a/ui/litellm-dashboard/next.config.mjs b/ui/litellm-dashboard/next.config.mjs index d4fdc7af36f..6b67fd70695 100644 --- a/ui/litellm-dashboard/next.config.mjs +++ b/ui/litellm-dashboard/next.config.mjs @@ -13,9 +13,11 @@ const nextConfig = { async rewrites() { return { beforeFiles: [ + // Every dashboard HTTP client sends Accept: application/json; page loads and RSC + // fetches do not. That is what keeps GET /lens (API) apart from /lens (page) in dev. { source: "/:path*", - has: [{ type: "header", key: "content-type", value: "application/json.*" }], + has: [{ type: "header", key: "accept", value: "application/json.*" }], destination: `${devProxyUrl}/:path*`, }, { source: "/ui/:path*", destination: "/:path*" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts index 60156d8ef25..c039b018b8f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.test.ts @@ -62,6 +62,7 @@ describe("makeSystemOneRequest", () => { const expectedRequest: Partial = { method: "POST", headers: { + Accept: "application/json", "Content-Type": "application/json", Authorization: "Bearer session-key", }, diff --git a/ui/litellm-dashboard/src/contexts/ReactQueryProvider.test.tsx b/ui/litellm-dashboard/src/contexts/ReactQueryProvider.test.tsx index c7fce0fd3bb..51a97f31e1c 100644 --- a/ui/litellm-dashboard/src/contexts/ReactQueryProvider.test.tsx +++ b/ui/litellm-dashboard/src/contexts/ReactQueryProvider.test.tsx @@ -16,6 +16,10 @@ describe("root query retry policy", () => { expect(await attempts(new ApiError("forbidden", 403, null))).toBe(1); }); + it("does not retry a malformed success response, which a retry cannot fix", async () => { + expect(await attempts(new ApiError("Expected JSON from /lens but the server returned text/html", 200, ""))).toBe(1); + }); + it("retries server and network errors", async () => { expect(await attempts(new ApiError("down", 503, null))).toBe(4); expect(await attempts(new TypeError("Failed to fetch"))).toBe(4); diff --git a/ui/litellm-dashboard/src/contexts/ReactQueryProvider.tsx b/ui/litellm-dashboard/src/contexts/ReactQueryProvider.tsx index 5e9b6bdd2d5..ddcdbecc9c5 100644 --- a/ui/litellm-dashboard/src/contexts/ReactQueryProvider.tsx +++ b/ui/litellm-dashboard/src/contexts/ReactQueryProvider.tsx @@ -5,8 +5,10 @@ import { ApiError } from "@/lib/http/client"; const MAX_RETRIES = 3; +const isRetryable = (error: unknown): boolean => !(error instanceof ApiError) || error.status >= 500; + export const shouldRetry = (failureCount: number, error: unknown): boolean => - !(error instanceof ApiError && error.status >= 400 && error.status < 500) && failureCount < MAX_RETRIES; + isRetryable(error) && failureCount < MAX_RETRIES; const queryClient = new QueryClient({ defaultOptions: { queries: { retry: shouldRetry } } }); diff --git a/ui/litellm-dashboard/src/lib/http/api.test.ts b/ui/litellm-dashboard/src/lib/http/api.test.ts index 2ebf959867e..3bccaca07dc 100644 --- a/ui/litellm-dashboard/src/lib/http/api.test.ts +++ b/ui/litellm-dashboard/src/lib/http/api.test.ts @@ -1,6 +1,8 @@ // @vitest-environment node import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { fetchClient } from "./api"; +import { ApiError } from "./client"; +import { shouldRetry } from "@/contexts/ReactQueryProvider"; import { registerAuthHeaderNameGetter, registerAuthTokenGetter, @@ -70,6 +72,61 @@ describe("typed api client middleware", () => { expect(requests[0].headers.get("x-litellm-key")).toBe("Bearer explicit-token"); }); + it("asks for JSON on a bodyless GET so the next dev rewrite routes it to the proxy, not the page", async () => { + const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] })); + + await fetchClient.GET("/lens", { fetch }); + + expect(requests[0].headers.get("Accept")).toBe("application/json"); + expect(requests[0].headers.get("Content-Type")).toBeNull(); + }); + + it("keeps an Accept header the caller set", async () => { + const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] })); + + await fetchClient.GET("/lens", { fetch, headers: { Accept: "text/event-stream" } }); + + expect(requests[0].headers.get("Accept")).toBe("text/event-stream"); + }); + + it("reports a non-JSON success body as an ApiError naming the path that react-query will not retry", async () => { + const onError = vi.fn(); + registerErrorHandler(onError); + const html = new Response("", { + status: 200, + headers: { "Content-Type": "text/html" }, + }); + const { fetch } = capturingFetch(html); + + const error = await fetchClient.GET("/lens", { fetch }).catch((e: unknown) => e); + + expect(error).toBeInstanceOf(ApiError); + expect((error as ApiError).message).toBe("Expected JSON from /lens but the server returned text/html"); + expect((error as ApiError).status).toBe(200); + expect(onError).toHaveBeenCalledWith("Expected JSON from /lens but the server returned text/html"); + expect(shouldRetry(0, error)).toBe(false); + }); + + it("accepts a JSON success body whose content type carries a charset", async () => { + const response = new Response(JSON.stringify({ data: [] }), { + status: 200, + headers: { "Content-Type": "application/json; charset=utf-8" }, + }); + const { fetch } = capturingFetch(response); + + const { data } = await fetchClient.GET("/model_group/info", { fetch }); + + expect(data).toEqual({ data: [] }); + }); + + it("accepts a JSON body served without a JSON content type", async () => { + const { fetch } = capturingFetch(new Response(JSON.stringify({ data: [] }), { status: 200 })); + + const { data } = await fetchClient.GET("/model_group/info", { fetch }); + + expect(data).toEqual({ data: [] }); + }); + it("omits the auth header when no token is set", async () => { const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] })); diff --git a/ui/litellm-dashboard/src/lib/http/api.ts b/ui/litellm-dashboard/src/lib/http/api.ts index a4fcc45d86b..560682ec312 100644 --- a/ui/litellm-dashboard/src/lib/http/api.ts +++ b/ui/litellm-dashboard/src/lib/http/api.ts @@ -13,15 +13,39 @@ const BaseAwareRequest = function (url: string, init?: RequestInit): Request { return new globalThis.Request(target, init); } as unknown as typeof Request; +const isJsonMediaType = (contentType: string): boolean => /[/+]json\b/i.test(contentType); + +const carriesJson = async (response: Response): Promise => { + const contentType = response.headers.get("content-type"); + if (contentType !== null && isJsonMediaType(contentType)) return true; + const text = await response.clone().text(); + if (!text) return true; + try { + JSON.parse(text); + return true; + } catch { + return false; + } +}; + const middleware: Middleware = { onRequest({ request }) { + if (!request.headers.has("Accept")) { + request.headers.set("Accept", "application/json"); + } const token = getAuthToken(); if (token && !request.headers.has(getAuthHeaderName())) { request.headers.set(getAuthHeaderName(), `Bearer ${token}`); } }, - async onResponse({ response }) { - if (response.ok) return response; + async onResponse({ request, response }) { + if (response.ok) { + if (await carriesJson(response)) return response; + const contentType = response.headers.get("content-type") ?? "an unknown content type"; + const message = `Expected JSON from ${new URL(request.url).pathname} but the server returned ${contentType}`; + reportError(message); + throw new ApiError(message, response.status, await response.clone().text()); + } const raw = await response.clone().text(); let body: unknown = raw; let message: string; @@ -43,9 +67,10 @@ const middleware: Middleware = { * * The base URL is injected, not fixed at import: every request is built against * whatever registerBaseUrlGetter supplies at call time (a split-origin proxy or - * worker URL), falling back to the current origin. The middleware injects the - * auth header and maps non-2xx responses to ApiError so query functions can just - * read `.data`. + * worker URL), falling back to the current origin. The middleware sends + * `Accept: application/json` (the next dev rewrite routes on it), injects the + * auth header, and maps non-2xx responses and non-JSON success bodies to + * ApiError so query functions can just read `.data`. */ export const fetchClient = createFetchClient({ Request: BaseAwareRequest, diff --git a/ui/litellm-dashboard/src/lib/http/client.test.ts b/ui/litellm-dashboard/src/lib/http/client.test.ts index 8c47ed70931..5a7487e7b33 100644 --- a/ui/litellm-dashboard/src/lib/http/client.test.ts +++ b/ui/litellm-dashboard/src/lib/http/client.test.ts @@ -35,6 +35,7 @@ describe("createApiClient", () => { expect(url).toBe("https://proxy.example/models?team=t1&page=2"); expect(init).toMatchObject({ method: "GET" }); expect(init.headers).toEqual({ + Accept: "application/json", "Content-Type": "application/json", "x-litellm-key": "Bearer sk-123", }); @@ -120,7 +121,7 @@ describe("createApiClient", () => { await client.get("/public/info"); const [, init] = fetchImpl.mock.calls[0]; - expect(init.headers).toEqual({ "Content-Type": "application/json" }); + expect(init.headers).toEqual({ Accept: "application/json", "Content-Type": "application/json" }); }); it("getBlob returns the response body as a Blob on success", async () => { diff --git a/ui/litellm-dashboard/src/lib/http/client.ts b/ui/litellm-dashboard/src/lib/http/client.ts index 0d39171ae6f..c698869239e 100644 --- a/ui/litellm-dashboard/src/lib/http/client.ts +++ b/ui/litellm-dashboard/src/lib/http/client.ts @@ -156,7 +156,7 @@ export function createApiClient(config: ApiClientConfig): ApiClient { const url = appendQuery(`${getBaseUrl()}${path}`, query); - const headers: Record = {}; + const headers: Record = { Accept: "application/json" }; if (rawBody === undefined) { headers["Content-Type"] = "application/json"; }