From 1224ad9b0ca5d1d59512d05dde7c04b86da91fa4 Mon Sep 17 00:00:00 2001 From: yuneng Date: Tue, 21 Jul 2026 19:04:48 +0000 Subject: [PATCH] fix(ui): clarify response cache vs provider prompt caching in logs and caching dashboard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ui/litellm-dashboard/eslint-suppressions.json | 2 +- .../_components/cache_dashboard.test.tsx | 7 +++ .../caching/_components/cache_dashboard.tsx | 10 ++++ .../src/components/leftnav.tsx | 8 ++- .../src/components/page_metadata.ts | 3 +- .../LogDetailContent.test.tsx | 33 ++++++++++- .../LogDetailsDrawer/LogDetailContent.tsx | 55 +++++++++++++++---- 7 files changed, 102 insertions(+), 16 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index d596897c4c9..01208e994a7 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2207,7 +2207,7 @@ }, "src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx": { "no-nested-ternary": { - "count": 4 + "count": 3 } }, "src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx index 17d14cd7fac..7a755a341a4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx @@ -69,6 +69,13 @@ describe("CacheDashboard cache analytics charts", () => { adminGlobalCacheActivity.mockResolvedValue(cacheActivity); }); + it("labels the dashboard as response caching and distinguishes it from prompt caching", async () => { + renderDashboard(); + + expect(await screen.findByRole("heading", { name: "Response Caching" })).toBeInTheDocument(); + expect(screen.getByText(/separate from provider prompt caching/i)).toBeInTheDocument(); + }); + it("renders both chart card titles", async () => { renderDashboard(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index b8e8dc8adb1..c8bdc1572b1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -260,6 +260,16 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole return ( +
+

+ Response Caching +

+ + Analytics and settings for LiteLLM's response cache, which returns a stored response for repeated + requests instead of calling the provider again. This is separate from provider prompt caching; prompt caching + is configured under Settings and its per-request activity shows up as prompt cache tokens on the Logs page + +
Cache Analytics diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index b0d534e9b7c..e666800da5b 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -244,7 +244,13 @@ const menuGroups: MenuGroup[] = [ icon: , external_url: "https://models.litellm.ai/cookbook", }, - { key: "caching", page: "caching", label: "Caching", icon: , roles: all_admin_roles }, + { + key: "caching", + page: "caching", + label: "Response Caching", + icon: , + roles: all_admin_roles, + }, { key: "experimental", page: "experimental", diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 845e868b917..5cbc2461d4a 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -30,7 +30,8 @@ export const pageDescriptions: Record = { api_ref: "Browse API documentation and endpoints", "model-hub-table": "Explore available AI models and providers", "learning-resources": "Access tutorials and documentation", - caching: "Configure response caching and coordination Redis settings", + caching: + "Configure LiteLLM response caching (exact-match / semantic) and coordination Redis settings; separate from provider prompt caching", "transform-request": "Set up request transformation rules", "cost-tracking": "Track and analyze API costs", "ui-theme": "Customize dashboard appearance", diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx index 85a38e26977..9f85131b83e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx @@ -255,6 +255,8 @@ describe("LogDetailContent", () => { expect(screen.getByText("2 masked")).toBeInTheDocument(); }); + const cacheHitTag = () => screen.getByText("Response Cache Hit").closest(".ant-descriptions-item") as HTMLElement; + it("should display cache hit information when cache_hit is true", () => { render( { />, ); - expect(screen.getByText("Cache Hit")).toBeInTheDocument(); - expect(screen.getByText("true")).toBeInTheDocument(); - expect(screen.getByText("Cache Read Tokens")).toBeInTheDocument(); + expect(screen.getByText("Response Cache Hit")).toBeInTheDocument(); + expect(within(cacheHitTag()).getByText("true").closest(".ant-tag")).toHaveClass("ant-tag-green"); + expect(screen.getByText("Prompt Cache Read Tokens")).toBeInTheDocument(); expect(screen.getByText("100")).toBeInTheDocument(); }); + it("should not render a red 'false' tag when only provider prompt caching is active", () => { + render( + , + ); + + const falseTag = within(cacheHitTag()).getByText("false").closest(".ant-tag") as HTMLElement; + expect(falseTag).not.toHaveClass("ant-tag-red"); + + expect(screen.getByText("Prompt Cache Read Tokens")).toBeInTheDocument(); + expect(screen.getByText("512")).toBeInTheDocument(); + expect(screen.getByText("Prompt Cache Creation Tokens")).toBeInTheDocument(); + expect(screen.getByText("128")).toBeInTheDocument(); + }); + it("should display LiteLLM Overhead when litellm_overhead_time_ms is in metadata", () => { render( + {label} + + + + + ); +} + function TagsSection({ tags }: { tags: Record }) { return (
@@ -291,8 +303,10 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: metadata.additional_usage_values.cache_read_input_tokens > 0); const cacheHitValue = String(logEntry.cache_hit ?? "None"); - const cacheHitColor = - cacheHitValue.toLowerCase() === "true" ? "green" : cacheHitValue.toLowerCase() === "false" ? "red" : "default"; + const cacheHitColor = cacheHitValue.toLowerCase() === "true" ? "green" : "default"; + + const cacheReadTokens = metadata?.additional_usage_values?.cache_read_input_tokens; + const cacheCreationTokens = metadata?.additional_usage_values?.cache_creation_input_tokens; const uncachedInputTokens = getUncachedInputTextTokens(metadata); const showAnthropicMessagesInputOutput = @@ -328,17 +342,38 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: {hasCacheActivity && ( <> - + + } + > {cacheHitValue} - {metadata?.additional_usage_values?.cache_read_input_tokens > 0 && ( - - {formatNumberWithCommas(metadata.additional_usage_values.cache_read_input_tokens)} + {cacheReadTokens > 0 && ( + + } + > + {formatNumberWithCommas(cacheReadTokens)} )} - {metadata?.additional_usage_values?.cache_creation_input_tokens > 0 && ( - - {formatNumberWithCommas(metadata.additional_usage_values.cache_creation_input_tokens)} + {cacheCreationTokens > 0 && ( + + } + > + {formatNumberWithCommas(cacheCreationTokens)} )}