diff --git a/litellm/proxy/analytics_endpoints/analytics_endpoints.py b/litellm/proxy/analytics_endpoints/analytics_endpoints.py index 4c1ff31e5a1..434657053d5 100644 --- a/litellm/proxy/analytics_endpoints/analytics_endpoints.py +++ b/litellm/proxy/analytics_endpoints/analytics_endpoints.py @@ -79,6 +79,7 @@ async def get_global_activity( sl."model", COUNT(*) AS total_rows, SUM(CASE WHEN sl."cache_hit" = 'True' THEN 1 ELSE 0 END) AS cache_hit_true_rows, + SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END) AS failed_rows, SUM(CASE WHEN sl."cache_hit" = 'True' THEN sl."completion_tokens" ELSE 0 END) AS cached_completion_tokens, SUM(CASE WHEN sl."cache_hit" != 'True' THEN sl."completion_tokens" ELSE 0 END) AS generated_completion_tokens FROM "LiteLLM_SpendLogs" sl diff --git a/tests/test_litellm/proxy/analytics_endpoints/__init__.py b/tests/test_litellm/proxy/analytics_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py b/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py new file mode 100644 index 00000000000..b90f925ffcf --- /dev/null +++ b/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py @@ -0,0 +1,53 @@ +""" +The cache dashboard buckets spend-log rows whose call_type is empty as "Unknown"; +those rows are failed requests. The activity SQL must therefore report a +failed_rows count per group so the UI can chart failures as their own series. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from litellm.proxy.analytics_endpoints.analytics_endpoints import get_global_activity + + +@pytest.fixture +def mock_prisma(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + return prisma + + +@pytest.mark.asyncio +async def test_cache_hits_query_counts_failed_rows_per_group(mock_prisma: MagicMock): + rows = [ + { + "api_key": "my-key-alias", + "call_type": "acompletion", + "model": "gpt-5.1", + "total_rows": 10, + "cache_hit_true_rows": 3, + "failed_rows": 2, + "cached_completion_tokens": 100, + "generated_completion_tokens": 900, + } + ] + mock_prisma.db.query_raw = AsyncMock(return_value=rows) + + response = await get_global_activity(start_date="2026-07-01", end_date="2026-07-27") + + assert response == rows + sql_query = mock_prisma.db.query_raw.call_args.args[0] + assert 'SUM(CASE WHEN sl."status" = \'failure\' THEN 1 ELSE 0 END) AS failed_rows' in sql_query + assert 'SUM(CASE WHEN sl."cache_hit" = \'True\' THEN 1 ELSE 0 END) AS cache_hit_true_rows' in sql_query + + +@pytest.mark.asyncio +async def test_cache_hits_requires_date_range(mock_prisma: MagicMock): + with pytest.raises(HTTPException) as exc_info: + await get_global_activity(start_date=None, end_date=None) + + assert exc_info.value.status_code == 400 + mock_prisma.db.query_raw.assert_not_called() diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 998422a78d2..407a7238870 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -152,7 +152,7 @@ "count": 1 }, "prefer-const": { - "count": 3 + "count": 1 }, "react-hooks/purity": { "count": 1 @@ -225,11 +225,6 @@ "count": 1 } }, - "src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/cost-tracking/_components/add_margin_form.tsx": { "local/filename-pascal-case": { "count": 1 @@ -4347,4 +4342,4 @@ "count": 1 } } -} +} \ No newline at end of file diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx index e1d02e9352d..3f927db7e21 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx @@ -21,6 +21,7 @@ const cacheActivity = [ call_type: "acompletion", total_rows: 1500, cache_hit_true_rows: 300, + failed_rows: 200, cached_completion_tokens: 12000, generated_completion_tokens: 48000, }, @@ -30,6 +31,7 @@ const cacheActivity = [ call_type: "aembedding", total_rows: 700, cache_hit_true_rows: 100, + failed_rows: 50, cached_completion_tokens: 2000, generated_completion_tokens: 9000, }, @@ -108,8 +110,13 @@ describe("CacheDashboard cache analytics charts", () => { expect(legendFillByCategory(requestsCard)).toEqual({ "LLM API requests": "var(--color-sky-500, #0ea5e9)", "Cache hit": "var(--color-teal-500, #14b8a6)", + "Failed requests": "var(--color-red-500, #ef4444)", }); - expect(barFills(requestsCard)).toEqual(["var(--color-sky-500, #0ea5e9)", "var(--color-teal-500, #14b8a6)"]); + expect(barFills(requestsCard)).toEqual([ + "var(--color-sky-500, #0ea5e9)", + "var(--color-teal-500, #14b8a6)", + "var(--color-red-500, #ef4444)", + ]); }); it("renders the tokens chart with each category legend-bound to its fill and stacked in order", async () => { @@ -133,18 +140,28 @@ describe("CacheDashboard cache analytics charts", () => { } }); - it("stacks the two categories into one column per call_type", async () => { + it("stacks all categories into one column per call_type", async () => { renderDashboard(); const { requestsCard, tokensCard } = await findChartCards(); - for (const card of [requestsCard, tokensCard]) { + const expectedRects = { requests: 6, tokens: 4 }; + for (const [card, rectCount] of [ + [requestsCard, expectedRects.requests], + [tokensCard, expectedRects.tokens], + ] as const) { const rects = Array.from(card.querySelectorAll("path.recharts-rectangle")); - expect(rects).toHaveLength(4); + expect(rects).toHaveLength(rectCount); const xPositions = rects.map((rect) => rect.getAttribute("d")?.split(",")[0]); expect(new Set(xPositions).size).toBe(2); } }); + it("keeps failed requests in the cache hit ratio denominator", async () => { + renderDashboard(); + + expect(await screen.findByText("18.18%")).toBeInTheDocument(); + }); + it("formats y-axis ticks with compact notation", async () => { renderDashboard(); const { requestsCard, tokensCard } = await findChartCards(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index 95b73d1aacb..616c7f05fa5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -23,6 +23,7 @@ import { adminGlobalCacheActivity, cachingHealthCheckCall } from "@/components/n // Import the new component import { CacheHealthTab } from "./cache_health"; +import { REQUEST_SERIES, summarizeCacheActivity, type CacheActivityRow, type CacheChartDatum } from "./cache_data"; import CacheSettings from "./cache_settings"; import CoordinationRedisSettings from "./coordination_redis_settings"; @@ -49,26 +50,6 @@ interface CachePageProps { premiumUser: boolean; } -interface cacheDataItem { - api_key: string; - model: string; - cache_hit_true_rows: number; - cached_completion_tokens: number; - total_rows: number; - generated_completion_tokens: number; - call_type: string; - - // Add other properties as needed -} - -type uiData = { - name: string; - "LLM API requests": number; - "Cache hit": number; - "Cached Completion Tokens": number; - "Generated Completion Tokens": number; -}; - interface CacheHealthResponse { status?: string; cache_type?: string; @@ -97,10 +78,10 @@ const deepParse = (input: any) => { }; const CacheDashboard: React.FC = ({ accessToken, token, userRole, userID, premiumUser }) => { - const [filteredData, setFilteredData] = useState([]); + const [filteredData, setFilteredData] = useState([]); const [selectedApiKeys, setSelectedApiKeys] = useState([]); const [selectedModels, setSelectedModels] = useState([]); - const [data, setData] = useState([]); + const [data, setData] = useState([]); const [cachedResponses, setCachedResponses] = useState("0"); const [cachedTokens, setCachedTokens] = useState("0"); const [cacheHitRatio, setCacheHitRatio] = useState("0"); @@ -150,7 +131,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole }; useEffect(() => { - let newData: cacheDataItem[] = data; + let newData: CacheActivityRow[] = data; if (selectedApiKeys.length > 0) { newData = newData.filter((item) => selectedApiKeys.includes(item.api_key)); } @@ -159,68 +140,18 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole newData = newData.filter((item) => selectedModels.includes(item.model)); } - /* - Data looks like this - [{"api_key":"sk-test-mock-key-001","call_type":"acompletion","model":"llama3-8b-8192","total_rows":13,"cache_hit_true_rows":0}, - {"api_key":"sk-test-mock-key-002","call_type":"None","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, - {"api_key":"sk-test-mock-key-123","call_type":"acompletion","model":"gpt-3.5-turbo","total_rows":19,"cache_hit_true_rows":0}, - {"api_key":"sk-test-mock-key-123","call_type":"aimage_generation","model":"","total_rows":3,"cache_hit_true_rows":0}, - {"api_key":"sk-test-mock-key-003","call_type":"None","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, - {"api_key":"sk-test-mock-key-004","call_type":"","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, - {"api_key":"sk-test-mock-key-005","call_type":"","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, - */ + const summary = summarizeCacheActivity(newData); - // What data we need for bar chat - // ui_data = [ - // { - // name: "Call Type", - // Cache hit: 20, - // LLM API requests: 10, - // } - // ] - - let llm_api_requests = 0; - let cache_hits = 0; - let cached_tokens = 0; - const processedData = newData.reduce((acc: uiData[], item) => { - if (!item.call_type) { - item.call_type = "Unknown"; - } - - llm_api_requests += (item.total_rows || 0) - (item.cache_hit_true_rows || 0); - cache_hits += item.cache_hit_true_rows || 0; - cached_tokens += item.cached_completion_tokens || 0; - - const existingItem = acc.find((i) => i.name === item.call_type); - if (existingItem) { - existingItem["LLM API requests"] += (item.total_rows || 0) - (item.cache_hit_true_rows || 0); - existingItem["Cache hit"] += item.cache_hit_true_rows || 0; - existingItem["Cached Completion Tokens"] += item.cached_completion_tokens || 0; - existingItem["Generated Completion Tokens"] += item.generated_completion_tokens || 0; - } else { - acc.push({ - name: item.call_type, - "LLM API requests": (item.total_rows || 0) - (item.cache_hit_true_rows || 0), - "Cache hit": item.cache_hit_true_rows || 0, - "Cached Completion Tokens": item.cached_completion_tokens || 0, - "Generated Completion Tokens": item.generated_completion_tokens || 0, - }); - } - return acc; - }, []); - - // set header cache statistics - setCachedResponses(valueFormatterNumbers(cache_hits)); - setCachedTokens(valueFormatterNumbers(cached_tokens)); - let allRequests = cache_hits + llm_api_requests; + setCachedResponses(valueFormatterNumbers(summary.cacheHits)); + setCachedTokens(valueFormatterNumbers(summary.cachedCompletionTokens)); + const allRequests = summary.cacheHits + summary.llmApiRequests + summary.failedRequests; if (allRequests > 0) { - let cache_hit_ratio = ((cache_hits / allRequests) * 100).toFixed(2); - setCacheHitRatio(cache_hit_ratio); + setCacheHitRatio(((summary.cacheHits / allRequests) * 100).toFixed(2)); } else { setCacheHitRatio("0"); } - setFilteredData(processedData); + setFilteredData(summary.chartData); }, [selectedApiKeys, selectedModels, dateValue, data]); const handleRefreshClick = () => { @@ -408,8 +339,8 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole stack={true} index="name" valueFormatter={valueFormatterNumbers} - categories={["LLM API requests", "Cache hit"]} - colors={["sky", "teal"]} + categories={[REQUEST_SERIES.apiRequests, REQUEST_SERIES.cacheHits, REQUEST_SERIES.failed]} + colors={["sky", "teal", "red"]} yAxisWidth={48} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.test.ts new file mode 100644 index 00000000000..fafbd00df8e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { summarizeCacheActivity, UNKNOWN_CALL_TYPE, type CacheActivityRow } from "./cache_data"; + +const row = (overrides: Partial): CacheActivityRow => ({ + api_key: "sk-1", + model: "gpt-5.1", + call_type: "acompletion", + total_rows: 0, + cache_hit_true_rows: 0, + ...overrides, +}); + +describe("summarizeCacheActivity", () => { + it("splits each call_type into api requests, cache hits, and failed requests", () => { + const summary = summarizeCacheActivity([row({ total_rows: 1000, cache_hit_true_rows: 300, failed_rows: 100 })]); + + expect(summary.chartData).toEqual([ + expect.objectContaining({ + name: "acompletion", + "LLM API requests": 600, + "Cache hit": 300, + "Failed requests": 100, + }), + ]); + expect(summary.llmApiRequests).toBe(600); + expect(summary.cacheHits).toBe(300); + expect(summary.failedRequests).toBe(100); + }); + + it("treats a missing failed_rows as zero so older proxy responses keep the old math", () => { + const summary = summarizeCacheActivity([row({ total_rows: 50, cache_hit_true_rows: 20 })]); + + expect(summary.chartData[0]["LLM API requests"]).toBe(30); + expect(summary.chartData[0]["Failed requests"]).toBe(0); + }); + + it("buckets rows with an empty call_type under Unknown", () => { + const summary = summarizeCacheActivity([ + row({ call_type: "", total_rows: 8, failed_rows: 8 }), + row({ call_type: "", api_key: "sk-2", total_rows: 3, failed_rows: 3 }), + ]); + + expect(summary.chartData).toHaveLength(1); + expect(summary.chartData[0]).toEqual( + expect.objectContaining({ name: UNKNOWN_CALL_TYPE, "Failed requests": 11, "LLM API requests": 0 }), + ); + }); + + it("merges rows sharing a call_type across keys and models", () => { + const summary = summarizeCacheActivity([ + row({ total_rows: 10, cache_hit_true_rows: 4, failed_rows: 1, cached_completion_tokens: 100 }), + row({ + api_key: "sk-2", + model: "claude-opus-4-8", + total_rows: 20, + cache_hit_true_rows: 6, + failed_rows: 2, + generated_completion_tokens: 500, + }), + ]); + + expect(summary.chartData).toEqual([ + { + name: "acompletion", + "LLM API requests": 17, + "Cache hit": 10, + "Failed requests": 3, + "Cached Completion Tokens": 100, + "Generated Completion Tokens": 500, + }, + ]); + expect(summary.cachedCompletionTokens).toBe(100); + }); + + it("returns empty totals for no rows", () => { + expect(summarizeCacheActivity([])).toEqual({ + chartData: [], + cacheHits: 0, + llmApiRequests: 0, + failedRequests: 0, + cachedCompletionTokens: 0, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.ts new file mode 100644 index 00000000000..d2006bf9a00 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.ts @@ -0,0 +1,73 @@ +export interface CacheActivityRow { + api_key: string; + model: string; + call_type: string; + total_rows: number; + cache_hit_true_rows: number; + failed_rows?: number; + cached_completion_tokens?: number; + generated_completion_tokens?: number; +} + +export const UNKNOWN_CALL_TYPE = "Unknown"; + +export const REQUEST_SERIES = { + apiRequests: "LLM API requests", + cacheHits: "Cache hit", + failed: "Failed requests", +} as const; + +export type CacheChartDatum = { + name: string; + [REQUEST_SERIES.apiRequests]: number; + [REQUEST_SERIES.cacheHits]: number; + [REQUEST_SERIES.failed]: number; + "Cached Completion Tokens": number; + "Generated Completion Tokens": number; +}; + +export interface CacheActivitySummary { + chartData: CacheChartDatum[]; + cacheHits: number; + llmApiRequests: number; + failedRequests: number; + cachedCompletionTokens: number; +} + +export function summarizeCacheActivity(rows: readonly CacheActivityRow[]): CacheActivitySummary { + const groups = new Map(); + + for (const row of rows) { + const name = row.call_type || UNKNOWN_CALL_TYPE; + const hits = row.cache_hit_true_rows || 0; + const failed = row.failed_rows || 0; + const apiRequests = (row.total_rows || 0) - hits - failed; + + const group = groups.get(name) ?? { + name, + [REQUEST_SERIES.apiRequests]: 0, + [REQUEST_SERIES.cacheHits]: 0, + [REQUEST_SERIES.failed]: 0, + "Cached Completion Tokens": 0, + "Generated Completion Tokens": 0, + }; + + groups.set(name, { + ...group, + [REQUEST_SERIES.apiRequests]: group[REQUEST_SERIES.apiRequests] + apiRequests, + [REQUEST_SERIES.cacheHits]: group[REQUEST_SERIES.cacheHits] + hits, + [REQUEST_SERIES.failed]: group[REQUEST_SERIES.failed] + failed, + "Cached Completion Tokens": group["Cached Completion Tokens"] + (row.cached_completion_tokens || 0), + "Generated Completion Tokens": group["Generated Completion Tokens"] + (row.generated_completion_tokens || 0), + }); + } + + const chartData = Array.from(groups.values()); + return { + chartData, + cacheHits: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.cacheHits], 0), + llmApiRequests: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.apiRequests], 0), + failedRequests: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.failed], 0), + cachedCompletionTokens: chartData.reduce((sum, g) => sum + g["Cached Completion Tokens"], 0), + }; +}