diff --git a/.gitignore b/.gitignore index b812d45e349..13f2202305d 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,4 @@ crash.*.log .coverage ui/litellm-dashboard/out/ +litellm.log diff --git a/litellm/proxy/analytics_endpoints/analytics_endpoints.py b/litellm/proxy/analytics_endpoints/analytics_endpoints.py index 434657053d5..cb22c468c1e 100644 --- a/litellm/proxy/analytics_endpoints/analytics_endpoints.py +++ b/litellm/proxy/analytics_endpoints/analytics_endpoints.py @@ -1,106 +1,61 @@ #### Analytics Endpoints ##### from datetime import datetime, timezone -from typing import List, Optional +from typing import Annotated import fastapi from fastapi import APIRouter, Depends, HTTPException, status from litellm.proxy._types import * +from litellm.proxy.analytics_endpoints.cache_activity import CacheActivityResponse, get_cache_activity from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router = APIRouter() +def _parse_date(value: str, param_name: str) -> datetime: + try: + return datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=timezone.utc) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": f"{param_name} must be a YYYY-MM-DD date, got {value!r}"}, + ) + + @router.get( "/global/activity/cache_hits", tags=["Budget & Spend Tracking"], dependencies=[Depends(user_api_key_auth)], - responses={ - 200: {"model": List[LiteLLM_SpendLogs]}, - }, + response_model=CacheActivityResponse, include_in_schema=False, ) async def get_global_activity( - start_date: Optional[str] = fastapi.Query( - default=None, - description="Time from which to start viewing spend", - ), - end_date: Optional[str] = fastapi.Query( - default=None, - description="Time till which to view spend", - ), -): + start_date: Annotated[str, fastapi.Query(description="Time from which to start viewing spend")], + end_date: Annotated[str, fastapi.Query(description="Time till which to view spend")], + key_aliases: Annotated[ + list[str] | None, fastapi.Query(description="Only include spend from these key aliases") + ] = None, + models: Annotated[list[str] | None, fastapi.Query(description="Only include spend for these models")] = None, +) -> CacheActivityResponse: """ - Get number of cache hits, vs misses - - { - "daily_data": [ - const chartdata = [ - { - date: 'Jan 22', - cache_hits: 10, - llm_api_calls: 2000 - }, - { - date: 'Jan 23', - cache_hits: 10, - llm_api_calls: 12 - }, - ], - "sum_cache_hits": 20, - "sum_llm_api_calls": 2012 - } + Cache activity for the Admin UI cache dashboard, aggregated per call_type: + cache hits vs successful LLM API requests vs failed requests, plus totals + for the stat cards and the available key-alias/model filter options. """ - - if start_date is None or end_date is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": "Please provide start_date and end_date"}, - ) - - start_date_obj = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=timezone.utc) - end_date_obj = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=timezone.utc) - from litellm.proxy.proxy_server import prisma_client - try: - if prisma_client is None: - raise ValueError( - "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" - ) - - sql_query = """ - SELECT - CASE - WHEN vt."key_alias" IS NOT NULL THEN vt."key_alias" - ELSE 'Unnamed Key' - END AS api_key, - sl."call_type", - sl."model", - COUNT(*) AS total_rows, - SUM(CASE WHEN sl."cache_hit" = 'True' THEN 1 ELSE 0 END) AS cache_hit_true_rows, - SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END) AS failed_rows, - SUM(CASE WHEN sl."cache_hit" = 'True' THEN sl."completion_tokens" ELSE 0 END) AS cached_completion_tokens, - SUM(CASE WHEN sl."cache_hit" != 'True' THEN sl."completion_tokens" ELSE 0 END) AS generated_completion_tokens - FROM "LiteLLM_SpendLogs" sl - LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token" - WHERE - sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') - AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') - GROUP BY - vt."key_alias", - sl."call_type", - sl."model" - """ - db_response = await prisma_client.db.query_raw(sql_query, start_date_obj, end_date_obj) - - if db_response is None: - return [] - - return db_response - - except Exception as e: + if prisma_client is None: raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": str(e)}, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "error": "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" + }, ) + + return await get_cache_activity( + prisma_client=prisma_client, + start_date=_parse_date(start_date, "start_date"), + end_date=_parse_date(end_date, "end_date"), + key_aliases=key_aliases or [], + models=models or [], + ) diff --git a/litellm/proxy/analytics_endpoints/cache_activity.py b/litellm/proxy/analytics_endpoints/cache_activity.py new file mode 100644 index 00000000000..6e20382f56e --- /dev/null +++ b/litellm/proxy/analytics_endpoints/cache_activity.py @@ -0,0 +1,137 @@ +import asyncio +import json +from datetime import datetime +from typing import TYPE_CHECKING, Sequence + +from pydantic import BaseModel, TypeAdapter + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +UNKNOWN_CALL_TYPE = "Unknown" + + +class CacheActivityGroup(BaseModel): + call_type: str + api_requests: int + cache_hits: int + failed_requests: int + cached_completion_tokens: int + generated_completion_tokens: int + + +class CacheActivityTotals(BaseModel): + api_requests: int + cache_hits: int + failed_requests: int + cached_completion_tokens: int + cache_hit_ratio: float + + +class CacheActivityFilterOptions(BaseModel): + key_aliases: list[str] + models: list[str] + + +class CacheActivityResponse(BaseModel): + groups: list[CacheActivityGroup] + totals: CacheActivityTotals + filter_options: CacheActivityFilterOptions + + +GROUPS_SQL = """ + SELECT + CASE WHEN sl."call_type" = '' THEN 'Unknown' ELSE sl."call_type" END AS call_type, + (COUNT(*) + - SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN 1 ELSE 0 END) + - SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END))::int AS api_requests, + SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN 1 ELSE 0 END)::int AS cache_hits, + SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END)::int AS failed_requests, + SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN sl."completion_tokens" ELSE 0 END)::int + AS cached_completion_tokens, + SUM(CASE WHEN COALESCE(sl."cache_hit", '') != 'True' THEN sl."completion_tokens" ELSE 0 END)::int + AS generated_completion_tokens + FROM "LiteLLM_SpendLogs" sl + LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token" + WHERE + sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') + AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') + AND ($3::jsonb = '[]'::jsonb + OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb))) + AND ($4::jsonb = '[]'::jsonb + OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb))) + GROUP BY 1 + ORDER BY (COUNT(*)) DESC +""" + +KEY_ALIAS_OPTIONS_SQL = """ + SELECT DISTINCT COALESCE(vt."key_alias", 'Unnamed Key') AS key_alias + FROM "LiteLLM_SpendLogs" sl + LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token" + WHERE + sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') + AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') + ORDER BY 1 +""" + +MODEL_OPTIONS_SQL = """ + SELECT DISTINCT sl."model" AS model + FROM "LiteLLM_SpendLogs" sl + WHERE + sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') + AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') + AND sl."model" != '' + ORDER BY 1 +""" + + +class _KeyAliasRow(BaseModel): + key_alias: str + + +class _ModelRow(BaseModel): + model: str + + +_groups_adapter = TypeAdapter(list[CacheActivityGroup]) +_key_alias_rows_adapter = TypeAdapter(list[_KeyAliasRow]) +_model_rows_adapter = TypeAdapter(list[_ModelRow]) + + +def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals: + api_requests = sum(group.api_requests for group in groups) + cache_hits = sum(group.cache_hits for group in groups) + failed_requests = sum(group.failed_requests for group in groups) + all_requests = api_requests + cache_hits + failed_requests + return CacheActivityTotals( + api_requests=api_requests, + cache_hits=cache_hits, + failed_requests=failed_requests, + cached_completion_tokens=sum(group.cached_completion_tokens for group in groups), + cache_hit_ratio=(cache_hits / all_requests) * 100 if all_requests > 0 else 0.0, + ) + + +async def get_cache_activity( + prisma_client: "PrismaClient", + start_date: datetime, + end_date: datetime, + key_aliases: Sequence[str], + models: Sequence[str], +) -> CacheActivityResponse: + group_rows, key_alias_rows, model_rows = await asyncio.gather( + prisma_client.db.query_raw( + GROUPS_SQL, start_date, end_date, json.dumps(list(key_aliases)), json.dumps(list(models)) + ), + prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date), + prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date), + ) + groups = _groups_adapter.validate_python(group_rows or []) + return CacheActivityResponse( + groups=groups, + totals=compute_totals(groups), + filter_options=CacheActivityFilterOptions( + key_aliases=[row.key_alias for row in _key_alias_rows_adapter.validate_python(key_alias_rows or [])], + models=[row.model for row in _model_rows_adapter.validate_python(model_rows or [])], + ), + ) diff --git a/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py b/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py index b90f925ffcf..c48b8cfd5a5 100644 --- a/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py +++ b/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py @@ -1,53 +1,133 @@ """ -The cache dashboard buckets spend-log rows whose call_type is empty as "Unknown"; -those rows are failed requests. The activity SQL must therefore report a -failed_rows count per group so the UI can chart failures as their own series. +The cache dashboard chart is fed by /global/activity/cache_hits. Aggregation +lives server-side: the SQL groups per call_type (splitting cache hits vs +successful vs failed requests; failed spend logs have call_type '' today and +must surface as 'Unknown'), and the endpoint returns chart-ready groups, +totals for the stat cards, and the filter options for the UI dropdowns. """ +import json from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException from litellm.proxy.analytics_endpoints.analytics_endpoints import get_global_activity +from litellm.proxy.analytics_endpoints.cache_activity import ( + GROUPS_SQL, + CacheActivityGroup, + compute_totals, +) + +GROUP_ROWS = [ + { + "call_type": "acompletion", + "api_requests": 1000, + "cache_hits": 300, + "failed_requests": 200, + "cached_completion_tokens": 12000, + "generated_completion_tokens": 48000, + }, + { + "call_type": "Unknown", + "api_requests": 0, + "cache_hits": 0, + "failed_requests": 110, + "cached_completion_tokens": 0, + "generated_completion_tokens": 0, + }, +] +KEY_ALIAS_ROWS = [{"key_alias": "Unnamed Key"}, {"key_alias": "my-key"}] +MODEL_ROWS = [{"model": "gpt-5.1"}] + + +def build_prisma(query_raw: AsyncMock) -> MagicMock: + prisma = MagicMock() + prisma.db.query_raw = query_raw + return prisma + + +def dispatching_query_raw() -> AsyncMock: + async def dispatch(sql: str, *params: object) -> list[dict[str, object]]: + if "GROUP BY" in sql: + return GROUP_ROWS + if "key_alias" in sql: + return KEY_ALIAS_ROWS + return MODEL_ROWS + + return AsyncMock(side_effect=dispatch) @pytest.fixture def mock_prisma(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - prisma = MagicMock() - prisma.db.query_raw = AsyncMock(return_value=[]) + prisma = build_prisma(dispatching_query_raw()) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) return prisma @pytest.mark.asyncio -async def test_cache_hits_query_counts_failed_rows_per_group(mock_prisma: MagicMock): - rows = [ - { - "api_key": "my-key-alias", - "call_type": "acompletion", - "model": "gpt-5.1", - "total_rows": 10, - "cache_hit_true_rows": 3, - "failed_rows": 2, - "cached_completion_tokens": 100, - "generated_completion_tokens": 900, - } - ] - mock_prisma.db.query_raw = AsyncMock(return_value=rows) +async def test_returns_groups_totals_and_filter_options(mock_prisma: MagicMock): + response = await get_global_activity( + start_date="2026-07-01", end_date="2026-07-27", key_aliases=[], models=[] + ) - response = await get_global_activity(start_date="2026-07-01", end_date="2026-07-27") - - assert response == rows - sql_query = mock_prisma.db.query_raw.call_args.args[0] - assert 'SUM(CASE WHEN sl."status" = \'failure\' THEN 1 ELSE 0 END) AS failed_rows' in sql_query - assert 'SUM(CASE WHEN sl."cache_hit" = \'True\' THEN 1 ELSE 0 END) AS cache_hit_true_rows' in sql_query + assert [group.call_type for group in response.groups] == ["acompletion", "Unknown"] + assert response.groups[0].api_requests == 1000 + assert response.groups[0].failed_requests == 200 + assert response.totals.api_requests == 1000 + assert response.totals.cache_hits == 300 + assert response.totals.failed_requests == 310 + assert response.totals.cached_completion_tokens == 12000 + assert response.totals.cache_hit_ratio == pytest.approx((300 / 1610) * 100) + assert response.filter_options.key_aliases == ["Unnamed Key", "my-key"] + assert response.filter_options.models == ["gpt-5.1"] @pytest.mark.asyncio -async def test_cache_hits_requires_date_range(mock_prisma: MagicMock): +async def test_filters_are_passed_to_sql_as_json_arrays(mock_prisma: MagicMock): + await get_global_activity( + start_date="2026-07-01", + end_date="2026-07-27", + key_aliases=["my-key"], + models=["gpt-5.1", "claude-opus-4-8"], + ) + + groups_call = next( + call for call in mock_prisma.db.query_raw.call_args_list if "GROUP BY" in call.args[0] + ) + assert groups_call.args[3] == json.dumps(["my-key"]) + assert groups_call.args[4] == json.dumps(["gpt-5.1", "claude-opus-4-8"]) + + +@pytest.mark.asyncio +async def test_rejects_malformed_dates_with_400(mock_prisma: MagicMock): with pytest.raises(HTTPException) as exc_info: - await get_global_activity(start_date=None, end_date=None) + await get_global_activity(start_date="07/01/2026", end_date="2026-07-27", key_aliases=[], models=[]) assert exc_info.value.status_code == 400 mock_prisma.db.query_raw.assert_not_called() + + +def test_totals_ratio_is_zero_without_requests(): + totals = compute_totals([]) + + assert totals.cache_hit_ratio == 0.0 + assert totals.api_requests == 0 + + +def test_totals_denominator_includes_failed_requests(): + group = CacheActivityGroup( + call_type="acompletion", + api_requests=60, + cache_hits=20, + failed_requests=20, + cached_completion_tokens=0, + generated_completion_tokens=0, + ) + + assert compute_totals([group]).cache_hit_ratio == pytest.approx(20.0) + + +def test_groups_sql_splits_failures_and_labels_empty_call_type_unknown(): + assert "SUM(CASE WHEN sl.\"status\" = 'failure' THEN 1 ELSE 0 END)" in GROUPS_SQL + assert "CASE WHEN sl.\"call_type\" = '' THEN 'Unknown' ELSE sl.\"call_type\" END" in GROUPS_SQL diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx index 3f927db7e21..fd8dd011b05 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx @@ -4,38 +4,50 @@ import { screen, waitFor, within } from "@testing-library/react"; import { renderWithProviders } from "../../../../../tests/test-utils"; import CacheDashboard from "./cache_dashboard"; -const { adminGlobalCacheActivity, cachingHealthCheckCall } = vi.hoisted(() => ({ - adminGlobalCacheActivity: vi.fn(), +const { useCacheActivity, cachingHealthCheckCall } = vi.hoisted(() => ({ + useCacheActivity: vi.fn(), cachingHealthCheckCall: vi.fn(), })); vi.mock("@/components/networking", () => ({ - adminGlobalCacheActivity, cachingHealthCheckCall, })); -const cacheActivity = [ - { - api_key: "sk-1", - model: "gpt-5.1", - call_type: "acompletion", - total_rows: 1500, - cache_hit_true_rows: 300, - failed_rows: 200, - cached_completion_tokens: 12000, - generated_completion_tokens: 48000, +vi.mock("@/app/(dashboard)/hooks/caching/useCacheActivity", () => ({ + useCacheActivity, +})); + +const cacheActivity = { + groups: [ + { + call_type: "acompletion", + api_requests: 1000, + cache_hits: 300, + failed_requests: 200, + cached_completion_tokens: 12000, + generated_completion_tokens: 48000, + }, + { + call_type: "aembedding", + api_requests: 550, + cache_hits: 100, + failed_requests: 50, + cached_completion_tokens: 2000, + generated_completion_tokens: 9000, + }, + ], + totals: { + api_requests: 1550, + cache_hits: 400, + failed_requests: 250, + cached_completion_tokens: 14000, + cache_hit_ratio: (400 / 2200) * 100, }, - { - api_key: "sk-2", - model: "text-embedding-3-large", - call_type: "aembedding", - total_rows: 700, - cache_hit_true_rows: 100, - failed_rows: 50, - cached_completion_tokens: 2000, - generated_completion_tokens: 9000, + filter_options: { + key_aliases: ["my-key", "Unnamed Key"], + models: ["gpt-5.1", "text-embedding-3-large"], }, -]; +}; const renderDashboard = () => renderWithProviders( @@ -77,7 +89,7 @@ const legendFillByCategory = (card: HTMLElement) => describe("CacheDashboard cache analytics charts", () => { beforeEach(() => { vi.clearAllMocks(); - adminGlobalCacheActivity.mockResolvedValue(cacheActivity); + useCacheActivity.mockReturnValue({ data: cacheActivity, refetch: vi.fn() }); }); it("renders both chart card titles", async () => { @@ -156,12 +168,23 @@ describe("CacheDashboard cache analytics charts", () => { } }); - it("keeps failed requests in the cache hit ratio denominator", async () => { + it("renders the server-computed cache hit ratio", async () => { renderDashboard(); expect(await screen.findByText("18.18%")).toBeInTheDocument(); }); + it("passes the date range and selected filters to the activity query", () => { + renderDashboard(); + + expect(useCacheActivity).toHaveBeenCalledWith({ + startDate: expect.stringMatching(/^\d{4}-\d{2}-\d{2}$/), + endDate: expect.stringMatching(/^\d{4}-\d{2}-\d{2}$/), + keyAliases: [], + models: [], + }); + }); + it("formats y-axis ticks with compact notation", async () => { renderDashboard(); const { requestsCard, tokensCard } = await findChartCards(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index 616c7f05fa5..47c266ceac0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -19,14 +19,29 @@ import { import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { RefreshCw } from "lucide-react"; -import { adminGlobalCacheActivity, cachingHealthCheckCall } from "@/components/networking"; +import { cachingHealthCheckCall } from "@/components/networking"; +import { useCacheActivity, type CacheActivityGroup } from "@/app/(dashboard)/hooks/caching/useCacheActivity"; // Import the new component import { CacheHealthTab } from "./cache_health"; -import { REQUEST_SERIES, summarizeCacheActivity, type CacheActivityRow, type CacheChartDatum } from "./cache_data"; import CacheSettings from "./cache_settings"; import CoordinationRedisSettings from "./coordination_redis_settings"; +const REQUEST_SERIES = { + apiRequests: "LLM API requests", + cacheHits: "Cache hit", + failed: "Failed requests", +} as const; + +const toChartDatum = (group: CacheActivityGroup) => ({ + name: group.call_type, + [REQUEST_SERIES.apiRequests]: group.api_requests, + [REQUEST_SERIES.cacheHits]: group.cache_hits, + [REQUEST_SERIES.failed]: group.failed_requests, + "Cached Completion Tokens": group.cached_completion_tokens, + "Generated Completion Tokens": group.generated_completion_tokens, +}); + const formatDateWithoutTZ = (date: Date | undefined) => { if (!date) return undefined; return date.toISOString().split("T")[0]; @@ -78,13 +93,8 @@ const deepParse = (input: any) => { }; const CacheDashboard: React.FC = ({ accessToken, token, userRole, userID, premiumUser }) => { - const [filteredData, setFilteredData] = useState([]); const [selectedApiKeys, setSelectedApiKeys] = useState([]); const [selectedModels, setSelectedModels] = useState([]); - const [data, setData] = useState([]); - const [cachedResponses, setCachedResponses] = useState("0"); - const [cachedTokens, setCachedTokens] = useState("0"); - const [cacheHitRatio, setCacheHitRatio] = useState("0"); const [dateValue, setDateValue] = useState({ from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), @@ -94,70 +104,24 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole const [lastRefreshed, setLastRefreshed] = useState(""); const [healthCheckResponse, setHealthCheckResponse] = useState(""); - useEffect(() => { - if (!accessToken || !dateValue) { - return; - } - const fetchData = async () => { - const response = await adminGlobalCacheActivity( - accessToken, - formatDateWithoutTZ(dateValue.from), - formatDateWithoutTZ(dateValue.to), - ); - setData(response); - }; - fetchData(); - - const currentDate = new Date(); - setLastRefreshed(currentDate.toLocaleString()); - }, [accessToken]); - - const uniqueApiKeys = Array.from(new Set(data.map((item) => item?.api_key ?? ""))); - const uniqueModels = Array.from(new Set(data.map((item) => item?.model ?? ""))); - const uniqueCallTypes = Array.from(new Set(data.map((item) => item?.call_type ?? ""))); - - const updateCachingData = async (startTime: Date | undefined, endTime: Date | undefined) => { - if (!startTime || !endTime || !accessToken) { - return; - } - - let new_cache_data = await adminGlobalCacheActivity( - accessToken, - formatDateWithoutTZ(startTime), - formatDateWithoutTZ(endTime), - ); - - setData(new_cache_data); - }; + const { data: activity, refetch } = useCacheActivity({ + startDate: formatDateWithoutTZ(dateValue.from), + endDate: formatDateWithoutTZ(dateValue.to), + keyAliases: selectedApiKeys, + models: selectedModels, + }); useEffect(() => { - let newData: CacheActivityRow[] = data; - if (selectedApiKeys.length > 0) { - newData = newData.filter((item) => selectedApiKeys.includes(item.api_key)); - } + setLastRefreshed(new Date().toLocaleString()); + }, []); - if (selectedModels.length > 0) { - newData = newData.filter((item) => selectedModels.includes(item.model)); - } - - const summary = summarizeCacheActivity(newData); - - setCachedResponses(valueFormatterNumbers(summary.cacheHits)); - setCachedTokens(valueFormatterNumbers(summary.cachedCompletionTokens)); - const allRequests = summary.cacheHits + summary.llmApiRequests + summary.failedRequests; - if (allRequests > 0) { - setCacheHitRatio(((summary.cacheHits / allRequests) * 100).toFixed(2)); - } else { - setCacheHitRatio("0"); - } - - setFilteredData(summary.chartData); - }, [selectedApiKeys, selectedModels, dateValue, data]); + const uniqueApiKeys = activity?.filter_options.key_aliases ?? []; + const uniqueModels = activity?.filter_options.models ?? []; + const chartData = (activity?.groups ?? []).map(toChartDatum); const handleRefreshClick = () => { - // Update the 'lastRefreshed' state to the current date and time - const currentDate = new Date(); - setLastRefreshed(currentDate.toLocaleString()); + refetch(); + setLastRefreshed(new Date().toLocaleString()); }; const runCachingHealthCheck = async () => { @@ -188,10 +152,12 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole } }; + const totals = activity?.totals; + const hasRequests = totals != null && totals.api_requests + totals.cache_hits + totals.failed_requests > 0; const statCards = [ - { label: "Cache Hit Ratio", value: `${cacheHitRatio}%` }, - { label: "Cache Hits", value: cachedResponses }, - { label: "Cached Completion Tokens", value: cachedTokens }, + { label: "Cache Hit Ratio", value: `${hasRequests ? totals.cache_hit_ratio.toFixed(2) : "0"}%` }, + { label: "Cache Hits", value: valueFormatterNumbers(totals?.cache_hits ?? 0) }, + { label: "Cached Completion Tokens", value: valueFormatterNumbers(totals?.cached_completion_tokens ?? 0) }, ]; return ( @@ -311,7 +277,6 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole value={dateValue} onValueChange={(value) => { setDateValue(value); - updateCachingData(value.from, value.to); }} /> @@ -335,7 +300,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole = ({ accessToken, token, userRole ): CacheActivityRow => ({ - api_key: "sk-1", - model: "gpt-5.1", - call_type: "acompletion", - total_rows: 0, - cache_hit_true_rows: 0, - ...overrides, -}); - -describe("summarizeCacheActivity", () => { - it("splits each call_type into api requests, cache hits, and failed requests", () => { - const summary = summarizeCacheActivity([row({ total_rows: 1000, cache_hit_true_rows: 300, failed_rows: 100 })]); - - expect(summary.chartData).toEqual([ - expect.objectContaining({ - name: "acompletion", - "LLM API requests": 600, - "Cache hit": 300, - "Failed requests": 100, - }), - ]); - expect(summary.llmApiRequests).toBe(600); - expect(summary.cacheHits).toBe(300); - expect(summary.failedRequests).toBe(100); - }); - - it("treats a missing failed_rows as zero so older proxy responses keep the old math", () => { - const summary = summarizeCacheActivity([row({ total_rows: 50, cache_hit_true_rows: 20 })]); - - expect(summary.chartData[0]["LLM API requests"]).toBe(30); - expect(summary.chartData[0]["Failed requests"]).toBe(0); - }); - - it("buckets rows with an empty call_type under Unknown", () => { - const summary = summarizeCacheActivity([ - row({ call_type: "", total_rows: 8, failed_rows: 8 }), - row({ call_type: "", api_key: "sk-2", total_rows: 3, failed_rows: 3 }), - ]); - - expect(summary.chartData).toHaveLength(1); - expect(summary.chartData[0]).toEqual( - expect.objectContaining({ name: UNKNOWN_CALL_TYPE, "Failed requests": 11, "LLM API requests": 0 }), - ); - }); - - it("merges rows sharing a call_type across keys and models", () => { - const summary = summarizeCacheActivity([ - row({ total_rows: 10, cache_hit_true_rows: 4, failed_rows: 1, cached_completion_tokens: 100 }), - row({ - api_key: "sk-2", - model: "claude-opus-4-8", - total_rows: 20, - cache_hit_true_rows: 6, - failed_rows: 2, - generated_completion_tokens: 500, - }), - ]); - - expect(summary.chartData).toEqual([ - { - name: "acompletion", - "LLM API requests": 17, - "Cache hit": 10, - "Failed requests": 3, - "Cached Completion Tokens": 100, - "Generated Completion Tokens": 500, - }, - ]); - expect(summary.cachedCompletionTokens).toBe(100); - }); - - it("returns empty totals for no rows", () => { - expect(summarizeCacheActivity([])).toEqual({ - chartData: [], - cacheHits: 0, - llmApiRequests: 0, - failedRequests: 0, - cachedCompletionTokens: 0, - }); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.ts deleted file mode 100644 index d2006bf9a00..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_data.ts +++ /dev/null @@ -1,73 +0,0 @@ -export interface CacheActivityRow { - api_key: string; - model: string; - call_type: string; - total_rows: number; - cache_hit_true_rows: number; - failed_rows?: number; - cached_completion_tokens?: number; - generated_completion_tokens?: number; -} - -export const UNKNOWN_CALL_TYPE = "Unknown"; - -export const REQUEST_SERIES = { - apiRequests: "LLM API requests", - cacheHits: "Cache hit", - failed: "Failed requests", -} as const; - -export type CacheChartDatum = { - name: string; - [REQUEST_SERIES.apiRequests]: number; - [REQUEST_SERIES.cacheHits]: number; - [REQUEST_SERIES.failed]: number; - "Cached Completion Tokens": number; - "Generated Completion Tokens": number; -}; - -export interface CacheActivitySummary { - chartData: CacheChartDatum[]; - cacheHits: number; - llmApiRequests: number; - failedRequests: number; - cachedCompletionTokens: number; -} - -export function summarizeCacheActivity(rows: readonly CacheActivityRow[]): CacheActivitySummary { - const groups = new Map(); - - for (const row of rows) { - const name = row.call_type || UNKNOWN_CALL_TYPE; - const hits = row.cache_hit_true_rows || 0; - const failed = row.failed_rows || 0; - const apiRequests = (row.total_rows || 0) - hits - failed; - - const group = groups.get(name) ?? { - name, - [REQUEST_SERIES.apiRequests]: 0, - [REQUEST_SERIES.cacheHits]: 0, - [REQUEST_SERIES.failed]: 0, - "Cached Completion Tokens": 0, - "Generated Completion Tokens": 0, - }; - - groups.set(name, { - ...group, - [REQUEST_SERIES.apiRequests]: group[REQUEST_SERIES.apiRequests] + apiRequests, - [REQUEST_SERIES.cacheHits]: group[REQUEST_SERIES.cacheHits] + hits, - [REQUEST_SERIES.failed]: group[REQUEST_SERIES.failed] + failed, - "Cached Completion Tokens": group["Cached Completion Tokens"] + (row.cached_completion_tokens || 0), - "Generated Completion Tokens": group["Generated Completion Tokens"] + (row.generated_completion_tokens || 0), - }); - } - - const chartData = Array.from(groups.values()); - return { - chartData, - cacheHits: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.cacheHits], 0), - llmApiRequests: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.apiRequests], 0), - failedRequests: chartData.reduce((sum, g) => sum + g[REQUEST_SERIES.failed], 0), - cachedCompletionTokens: chartData.reduce((sum, g) => sum + g["Cached Completion Tokens"], 0), - }; -} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/caching/useCacheActivity.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/caching/useCacheActivity.test.ts new file mode 100644 index 00000000000..1a3baee1ce3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/caching/useCacheActivity.test.ts @@ -0,0 +1,72 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useCacheActivity, type CacheActivityParams } from "./useCacheActivity"; + +const useQueryMock = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + $api: { useQuery: (...args: unknown[]) => useQueryMock(...args) }, +})); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +const params: CacheActivityParams = { + startDate: "2026-07-20", + endDate: "2026-07-27", + keyAliases: ["my-key"], + models: ["gpt-5.1"], +}; + +const lastCallOptions = (): { enabled: boolean } => { + const calls = useQueryMock.mock.calls; + return calls[calls.length - 1][3] as { enabled: boolean }; +}; + +describe("useCacheActivity", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + mockUseAuthorized.mockReturnValue({ accessToken: "test-access-token" }); + }); + + it("queries GET /global/activity/cache_hits with dates and filters as query params", () => { + renderHook(() => useCacheActivity(params)); + + expect(useQueryMock).toHaveBeenCalledWith( + "get", + "/global/activity/cache_hits", + { + params: { + query: { + start_date: "2026-07-20", + end_date: "2026-07-27", + key_aliases: ["my-key"], + models: ["gpt-5.1"], + }, + }, + }, + expect.any(Object), + ); + }); + + it("enables the query when authorized and both dates are set", () => { + renderHook(() => useCacheActivity(params)); + + expect(lastCallOptions().enabled).toBe(true); + }); + + it("disables the query without an access token", () => { + mockUseAuthorized.mockReturnValue({ accessToken: null }); + renderHook(() => useCacheActivity(params)); + + expect(lastCallOptions().enabled).toBe(false); + }); + + it("disables the query while the date range is incomplete", () => { + renderHook(() => useCacheActivity({ ...params, endDate: undefined })); + + expect(lastCallOptions().enabled).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/caching/useCacheActivity.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/caching/useCacheActivity.ts new file mode 100644 index 00000000000..af4486ad33b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/caching/useCacheActivity.ts @@ -0,0 +1,32 @@ +import { $api } from "@/lib/http/api"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import type { components } from "@/lib/http/schema"; + +export type CacheActivityResponse = components["schemas"]["CacheActivityResponse"]; +export type CacheActivityGroup = components["schemas"]["CacheActivityGroup"]; + +export interface CacheActivityParams { + startDate: string | undefined; + endDate: string | undefined; + keyAliases: string[]; + models: string[]; +} + +export const useCacheActivity = ({ startDate, endDate, keyAliases, models }: CacheActivityParams) => { + const { accessToken } = useAuthorized(); + return $api.useQuery( + "get", + "/global/activity/cache_hits", + { + params: { + query: { + start_date: startDate ?? "", + end_date: endDate ?? "", + key_aliases: keyAliases, + models, + }, + }, + }, + { enabled: Boolean(accessToken && startDate && endDate) }, + ); +}; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 576e16cbb37..4abe3a6a154 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2094,42 +2094,6 @@ export const adminGlobalActivity = async ( } }; -export const adminGlobalCacheActivity = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/activity/cache_hits` : `/global/activity/cache_hits`; - - if (startTime && endTime) { - url += `?start_date=${startTime}&end_date=${endTime}`; - } - - const requestOptions = { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - }, - }; - - const response = await fetch(url, requestOptions); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to fetch spend data:", error); - throw error; - } -}; - export const adminGlobalActivityPerModel = async ( accessToken: string, startTime: string | undefined, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index fc08864f19f..80ea068cb5e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4356,25 +4356,9 @@ export interface paths { }; /** * Get Global Activity - * @description Get number of cache hits, vs misses - * - * { - * "daily_data": [ - * const chartdata = [ - * { - * date: 'Jan 22', - * cache_hits: 10, - * llm_api_calls: 2000 - * }, - * { - * date: 'Jan 23', - * cache_hits: 10, - * llm_api_calls: 12 - * }, - * ], - * "sum_cache_hits": 20, - * "sum_llm_api_calls": 2012 - * } + * @description Cache activity for the Admin UI cache dashboard, aggregated per call_type: + * cache hits vs successful LLM API requests vs failed requests, plus totals + * for the stat cards and the available key-alias/model filter options. */ get: operations["get_global_activity_global_activity_cache_hits_get"]; put?: never; @@ -21741,6 +21725,48 @@ export interface components { /** Total Requested */ total_requested: number; }; + /** CacheActivityFilterOptions */ + CacheActivityFilterOptions: { + /** Key Aliases */ + key_aliases: string[]; + /** Models */ + models: string[]; + }; + /** CacheActivityGroup */ + CacheActivityGroup: { + /** Api Requests */ + api_requests: number; + /** Cache Hits */ + cache_hits: number; + /** Cached Completion Tokens */ + cached_completion_tokens: number; + /** Call Type */ + call_type: string; + /** Failed Requests */ + failed_requests: number; + /** Generated Completion Tokens */ + generated_completion_tokens: number; + }; + /** CacheActivityResponse */ + CacheActivityResponse: { + filter_options: components["schemas"]["CacheActivityFilterOptions"]; + /** Groups */ + groups: components["schemas"]["CacheActivityGroup"][]; + totals: components["schemas"]["CacheActivityTotals"]; + }; + /** CacheActivityTotals */ + CacheActivityTotals: { + /** Api Requests */ + api_requests: number; + /** Cache Hit Ratio */ + cache_hit_ratio: number; + /** Cache Hits */ + cache_hits: number; + /** Cached Completion Tokens */ + cached_completion_tokens: number; + /** Failed Requests */ + failed_requests: number; + }; /** CachePingResponse */ CachePingResponse: { /** Cache Type */ @@ -40618,11 +40644,15 @@ export interface operations { }; get_global_activity_global_activity_cache_hits_get: { parameters: { - query?: { + query: { /** @description Time from which to start viewing spend */ - start_date?: string | null; + start_date: string; /** @description Time till which to view spend */ - end_date?: string | null; + end_date: string; + /** @description Only include spend from these key aliases */ + key_aliases?: string[] | null; + /** @description Only include spend for these models */ + models?: string[] | null; }; header?: never; path?: never; @@ -40636,7 +40666,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["LiteLLM_SpendLogs"][]; + "application/json": components["schemas"]["CacheActivityResponse"]; }; }; /** @description Validation Error */