From 94b2fd827b4ea2678fa88ac0788e948f3b899348 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 19 Sep 2026 17:02:56 -0700 Subject: [PATCH] feat(ui): show prompt caching requests and net savings --- backend/routes/allowlist.py | 1 + .../prompt_caching_requests.py | 184 ++++++++++ litellm/proxy/proxy_server.py | 4 + litellm/proxy/spend_tracking/savings.py | 75 ++-- .../prompt_caching_requests.py | 35 ++ .../test_prompt_caching_requests.py | 321 ++++++++++++++++++ .../proxy/spend_tracking/test_savings.py | 37 ++ .../_components/CacheLeakageCard.tsx | 6 +- .../CostOptimizationView.activity.test.tsx | 1 + ...tCachingRequestsTable.integration.test.tsx | 248 ++++++++++++++ .../PromptCachingRequestsTable.tsx | 186 ++++++++++ .../_components/PromptCachingTab.test.tsx | 23 +- .../_components/PromptCachingTab.tsx | 7 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 95 ++++++ 14 files changed, 1195 insertions(+), 28 deletions(-) create mode 100644 litellm/proxy/management_endpoints/prompt_caching_requests.py create mode 100644 litellm/types/management_endpoints/prompt_caching_requests.py create mode 100644 tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.tsx diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 00c4e0070e6..c7f389c36a4 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -51,6 +51,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/cache_settings", "/coordination_redis/", "/cost_tracking", + "/cost_optimization/", "/cost/", "/credentials", "/credential", diff --git a/litellm/proxy/management_endpoints/prompt_caching_requests.py b/litellm/proxy/management_endpoints/prompt_caching_requests.py new file mode 100644 index 00000000000..41255bd49b8 --- /dev/null +++ b/litellm/proxy/management_endpoints/prompt_caching_requests.py @@ -0,0 +1,184 @@ +from collections.abc import Callable, Mapping +from datetime import datetime, timezone +from types import MappingProxyType +from typing import TYPE_CHECKING, Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Json, TypeAdapter + +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth, user_api_key_has_admin_view +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.spend_tracking.savings import ( + extract_cache_creation_tokens, + extract_cache_read_tokens, + marks_gateway_injection, + prompt_caching_savings_for_request, +) +from litellm.proxy.spend_tracking.spend_tracking_utils import ( + _query_raw_rows, # pyright: ignore[reportPrivateUsage] # existing typed spend-query adapter; rows validated below +) +from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY +from litellm.types.management_endpoints.prompt_caching_requests import ( + PromptCachingRequest, + PromptCachingRequestCursor, + PromptCachingRequestFilter, + PromptCachingRequestsResponse, +) + +if TYPE_CHECKING: + from litellm.router import Router + +router: Final = APIRouter() + + +def _numeric_token_sql(path: str) -> str: + value: Final = f"metadata #> '{{usage_object,{path}}}'" + return ( + f"CASE WHEN jsonb_typeof({value}) = 'number' THEN ({value} #>> '{{}}')::numeric " + f"WHEN {value} = 'true'::jsonb THEN 1 WHEN {value} = 'false'::jsonb THEN 0 END" + ) + + +def _cache_tokens_sql(*paths: str) -> str: + candidates: Final = ", ".join(f"NULLIF(({_numeric_token_sql(path)}), 0)" for path in paths) + return f"TRUNC(COALESCE({candidates}, 0))" + + +_CACHE_READ_SQL: Final = _cache_tokens_sql("cache_read_input_tokens", "prompt_tokens_details,cached_tokens") +_CACHE_CREATION_SQL: Final = _cache_tokens_sql( + "cache_creation_input_tokens", + "prompt_tokens_details,cache_write_tokens", + "prompt_tokens_details,cache_creation_tokens", +) +_GATEWAY_INJECTED_SQL: Final = ( + f"(jsonb_typeof(metadata->'{GATEWAY_INJECTED_CACHE_METADATA_KEY}') = 'string' " + f"AND (metadata->>'{GATEWAY_INJECTED_CACHE_METADATA_KEY}' = '' " + f"OR metadata->>'{GATEWAY_INJECTED_CACHE_METADATA_KEY}' = model_id))" +) +_FILTER_SQL: Final = MappingProxyType( + { + "all": f"({_GATEWAY_INJECTED_SQL} OR {_CACHE_READ_SQL} > 0 OR {_CACHE_CREATION_SQL} > 0)", + "injected": _GATEWAY_INJECTED_SQL, + "hits": f"{_CACHE_READ_SQL} > 0", + } +) + + +def prompt_caching_requests_sql(filter: PromptCachingRequestFilter) -> str: + return f""" + SELECT request_id, "startTime" AS start_time, "endTime" AS end_time, + model, model_id, custom_llm_provider, spend, + CASE WHEN jsonb_typeof(metadata->'usage_object') = 'object' + THEN metadata->'usage_object' END AS usage_object, + CASE WHEN jsonb_typeof(metadata->'cost_breakdown') = 'object' + THEN metadata->'cost_breakdown' END AS cost_breakdown, + CASE WHEN jsonb_typeof(metadata->'{GATEWAY_INJECTED_CACHE_METADATA_KEY}') = 'string' + THEN metadata->>'{GATEWAY_INJECTED_CACHE_METADATA_KEY}' END AS gateway_marker + FROM "LiteLLM_SpendLogs" + WHERE "startTime" >= ($1::text::timestamptz AT TIME ZONE 'UTC') + AND "startTime" <= ($2::text::timestamptz AT TIME ZONE 'UTC') + AND COALESCE(LOWER(cache_hit), 'false') != 'true' + AND {_FILTER_SQL[filter]} + AND ($4::text::timestamptz IS NULL OR + ("startTime", request_id) < (($4::text::timestamptz AT TIME ZONE 'UTC'), $5::text)) + ORDER BY "startTime" DESC, request_id DESC + LIMIT $3::integer + """ + + +class _PromptCachingRow(BaseModel): + request_id: str + start_time: datetime + end_time: datetime + model: str + model_id: str | None + custom_llm_provider: str | None + spend: float + usage_object: Json[Mapping[str, object]] | Mapping[str, object] | None + cost_breakdown: Json[Mapping[str, object]] | Mapping[str, object] | None + gateway_marker: str | None + + +_REQUEST_ROWS: Final = TypeAdapter(tuple[_PromptCachingRow, ...]) + + +def _request_result(row: _PromptCachingRow, llm_router: "Callable[[], Router | None]") -> PromptCachingRequest: + return PromptCachingRequest( + request_id=row.request_id, + start_time=row.start_time.replace(tzinfo=timezone.utc) if row.start_time.tzinfo is None else row.start_time, + model=row.model, + gateway_injected=marks_gateway_injection( + MappingProxyType({GATEWAY_INJECTED_CACHE_METADATA_KEY: row.gateway_marker}), row.model_id + ), + cache_read_tokens=extract_cache_read_tokens(row.usage_object), + cache_creation_tokens=extract_cache_creation_tokens(row.usage_object), + spend=row.spend, + net_savings=prompt_caching_savings_for_request( + model=row.model, + custom_llm_provider=row.custom_llm_provider, + usage_object=row.usage_object, + model_id=row.model_id, + llm_router=llm_router, + cost_breakdown=row.cost_breakdown, + billed_at=row.end_time, + ), + ) + + +@router.get( + "/cost_optimization/prompt_caching/requests", + tags=["Cost Optimization"], # mutable-ok: FastAPI's route API requires a list + response_model=PromptCachingRequestsResponse, +) +async def get_prompt_caching_requests( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: datetime, + end_date: datetime, + page_size: Annotated[int, Query(ge=1, le=100)] = 50, + filter: PromptCachingRequestFilter = "all", + cursor_start_time: datetime | None = None, + cursor_request_id: Annotated[str | None, Query(min_length=1)] = None, +) -> PromptCachingRequestsResponse: + from litellm.proxy.proxy_server import llm_router, prisma_client + + if not user_api_key_has_admin_view(user_api_key_dict): + raise HTTPException(status_code=403, detail="Only proxy admin roles can view prompt caching requests") + if (cursor_start_time is None) != (cursor_request_id is None): + raise HTTPException(status_code=400, detail="cursor_start_time and cursor_request_id must be provided together") + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + start: Final = start_date.replace(tzinfo=timezone.utc) if start_date.tzinfo is None else start_date + end: Final = end_date.replace(tzinfo=timezone.utc) if end_date.tzinfo is None else end_date + if end < start: + raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date") + cursor_time: Final = ( + cursor_start_time.replace(tzinfo=timezone.utc) + if cursor_start_time is not None and cursor_start_time.tzinfo is None + else cursor_start_time + ) + rows: Final = _REQUEST_ROWS.validate_python( + await _query_raw_rows( + prisma_client, + prompt_caching_requests_sql(filter), + start.isoformat(), + end.isoformat(), + page_size + 1, + cursor_time.isoformat() if cursor_time is not None else None, + cursor_request_id, + ) + or () + ) + + def current_router() -> "Router | None": + return llm_router + + requests: Final = tuple(_request_result(row, current_router) for row in rows[:page_size]) + has_more: Final = len(rows) > page_size + return PromptCachingRequestsResponse( + requests=requests, + page_size=page_size, + has_more=has_more, + next_cursor=PromptCachingRequestCursor(start_time=requests[-1].start_time, request_id=requests[-1].request_id) + if has_more + else None, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af25d418a63..f4a56e225cc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -587,6 +587,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) +from litellm.proxy.management_endpoints.prompt_caching_requests import ( + router as prompt_caching_requests_router, +) from litellm.proxy.management_endpoints.router_settings_endpoints import ( router as router_settings_router, ) @@ -19183,6 +19186,7 @@ app.include_router(workflow_management_router) app.include_router(memory_router) app.include_router(plugin_router) app.include_router(cost_tracking_settings_router) +app.include_router(prompt_caching_requests_router) app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index b7a2ac62844..fbcf9c78d3e 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -578,6 +578,56 @@ def autorouter_savings_for_logging_payload( ) +def _request_savings_pricing( + model: str | None, + custom_llm_provider: str | None, + model_id: str | None, + llm_router: "Callable[[], Router | None] | None", +) -> tuple[str | None, ModelInfo | None]: + router_instance: Final = llm_router() if llm_router else None + identity: Final = _resolve_model(model, custom_llm_provider) + pricing: Final = _effective_model_info(router_instance, model_id, model or "") or ( + _model_info(identity) if identity else None + ) + return identity.provider if identity else custom_llm_provider, pricing + + +def _prompt_caching_savings( + pricing: ModelInfo | None, + provider: str | None, + usage_object: Mapping[str, object] | None, + cost_breakdown: Mapping[str, object] | None, + billed_at: datetime | str | None, +) -> float | None: + usage: Final = _usage_from_spend_log(usage_object) + if pricing is None or usage is None: + return None + basis: Final = _pricing_basis(cost_breakdown) + result: Final = calculate_prompt_caching_savings( + model_info=pricing, + usage=usage, + custom_llm_provider=provider, + service_tier=basis.service_tier, + data_residency=basis.data_residency, + vertex_location=basis.vertex_location, + billed_at=_coerce_billed_at(billed_at), + ) + return result if isfinite(result) else None + + +def prompt_caching_savings_for_request( + model: str | None, + custom_llm_provider: str | None, + usage_object: Mapping[str, object] | None, + model_id: str | None = None, + llm_router: "Callable[[], Router | None] | None" = None, + cost_breakdown: Mapping[str, object] | None = None, + billed_at: datetime | str | None = None, +) -> float | None: + request_pricing: Final = _request_savings_pricing(model, custom_llm_provider, model_id, llm_router) + return _prompt_caching_savings(request_pricing[1], request_pricing[0], usage_object, cost_breakdown, billed_at) + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, @@ -639,29 +689,12 @@ def compute_savings_spend( # Deployment rates when the request came through one, public rates otherwise -- # `_effective_model_info` merges a deployment's configured prices over the built-in # map, so a negotiated price is not silently replaced by the list rate. - router_instance: Router | None = llm_router() if llm_router else None - identity: Final = _resolve_model(model, custom_llm_provider) - pricing: Final = _effective_model_info(router_instance, model_id, model or "") or ( - _model_info(identity) if identity else None - ) + request_pricing: Final = _request_savings_pricing(model, custom_llm_provider, model_id, llm_router) + provider: Final = request_pricing[0] + pricing: Final = request_pricing[1] input_cost: Final = (_get_cost_per_unit(pricing, "input_cost_per_token") or 0.0) if pricing else 0.0 compression: Final = max(compression_saved_tokens, 0) * input_cost - usage: Final = _usage_from_spend_log(usage_object) - basis: Final = _pricing_basis(cost_breakdown) - billed_at_datetime: Final = _coerce_billed_at(billed_at) - prompt_caching: Final = ( - calculate_prompt_caching_savings( - model_info=pricing, - usage=usage, - custom_llm_provider=identity.provider if identity else custom_llm_provider, - service_tier=basis.service_tier, - data_residency=basis.data_residency, - vertex_location=basis.vertex_location, - billed_at=billed_at_datetime, - ) - if pricing is not None and usage is not None - else 0.0 - ) + prompt_caching: Final = _prompt_caching_savings(pricing, provider, usage_object, cost_breakdown, billed_at) or 0.0 gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0 # The figure the logging path recorded wins, before the usage gate on purpose: a row diff --git a/litellm/types/management_endpoints/prompt_caching_requests.py b/litellm/types/management_endpoints/prompt_caching_requests.py new file mode 100644 index 00000000000..e72183a113b --- /dev/null +++ b/litellm/types/management_endpoints/prompt_caching_requests.py @@ -0,0 +1,35 @@ +from datetime import datetime +from typing import Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict + +PromptCachingRequestFilter: TypeAlias = Literal["all", "injected", "hits"] + + +class PromptCachingRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + request_id: str + start_time: datetime + model: str + gateway_injected: bool + cache_read_tokens: int + cache_creation_tokens: int + spend: float + net_savings: float | None + + +class PromptCachingRequestCursor(BaseModel): + model_config = ConfigDict(frozen=True) + + start_time: datetime + request_id: str + + +class PromptCachingRequestsResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + requests: tuple[PromptCachingRequest, ...] + page_size: int + has_more: bool + next_cursor: PromptCachingRequestCursor | None diff --git a/tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py b/tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py new file mode 100644 index 00000000000..0995de6c39d --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py @@ -0,0 +1,321 @@ +import json +from collections.abc import AsyncIterator, Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from typing import Final + +import httpx +import psycopg +import pytest +import pytest_asyncio +from fastapi import FastAPI +from prisma import Prisma +from pydantic import TypeAdapter +from pytest_postgresql import factories + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_endpoints.prompt_caching_requests import router +from litellm.proxy.spend_tracking.savings import ( + extract_cache_creation_tokens, + extract_cache_read_tokens, + marks_gateway_injection, +) +from litellm.types.management_endpoints.prompt_caching_requests import ( + PromptCachingRequestFilter, + PromptCachingRequestsResponse, +) + +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + +_cache_postgresql_proc: Final = factories.postgresql_proc() # pyright: ignore[reportUnknownMemberType] # third-party fixture factory has incomplete callable types +_cache_postgresql: Final = factories.postgresql("_cache_postgresql_proc") +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_JSON_ROWS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_START: Final = "2026-09-01T00:00:00Z" +_END: Final = "2026-09-02T00:00:00Z" +_URL: Final = "/cost_optimization/prompt_caching/requests" +_MODEL: Final = "claude-sonnet-5" +_MARKER: Final = "litellm_gateway_injected_cache" +_DDL: Final = """ + CREATE TABLE "LiteLLM_SpendLogs" ( + request_id TEXT PRIMARY KEY, "startTime" TIMESTAMP, "endTime" TIMESTAMP, + model TEXT, model_id TEXT, custom_llm_provider TEXT, spend DOUBLE PRECISION, + metadata JSONB, cache_hit TEXT + ) +""" + + +@dataclass(frozen=True) +class _Case: + request_id: str + metadata: Mapping[str, object] + cache_hit: str | None = None + start_time: datetime = datetime(2026, 9, 1, 12, 0, 0, 123456) + + def matches(self, filter: PromptCachingRequestFilter) -> bool: + if self.cache_hit is not None and self.cache_hit.lower() == "true": + return False + if not datetime(2026, 9, 1) <= self.start_time <= datetime(2026, 9, 2): + return False + usage: Final = self.metadata.get("usage_object") + normalized: Final = _JSON_OBJECT.validate_python(usage) if isinstance(usage, Mapping) else None + injected: Final = marks_gateway_injection(self.metadata, "dep-a") + reads: Final = extract_cache_read_tokens(normalized) + writes: Final = extract_cache_creation_tokens(normalized) + match filter: + case "injected": + return injected + case "hits": + return reads > 0 + case "all": + return injected or reads > 0 or writes > 0 + + +_CASES: Final = ( + _Case("injected-empty", {_MARKER: ""}), + _Case("injected-deployment", {_MARKER: "dep-a"}), + _Case("wrong-deployment", {_MARKER: "dep-b"}), + _Case("legacy-read", {"usage_object": {"cache_read_input_tokens": 100}}), + _Case("nested-read", {"usage_object": {"prompt_tokens_details": {"cached_tokens": 100}}}), + _Case("write", {"usage_object": {"cache_creation_input_tokens": 100}}), + _Case("nested-write", {"usage_object": {"prompt_tokens_details": {"cache_write_tokens": 100}}}), + _Case("nested-creation", {"usage_object": {"prompt_tokens_details": {"cache_creation_tokens": 100}}}), + _Case( + "top-precedence", + {"usage_object": {"cache_read_input_tokens": -2, "prompt_tokens_details": {"cached_tokens": 100}}}, + ), + _Case( + "zero-fallback", + {"usage_object": {"cache_read_input_tokens": 0, "prompt_tokens_details": {"cached_tokens": 100}}}, + ), + _Case( + "fractional-precedence", + {"usage_object": {"cache_read_input_tokens": 0.5, "prompt_tokens_details": {"cached_tokens": 100}}}, + ), + _Case("malformed-number", {"usage_object": {"cache_read_input_tokens": "100"}}), + _Case("malformed-container", {"usage_object": [100]}), + _Case("boolean-number", {"usage_object": {"cache_read_input_tokens": True}}), + _Case("boolean-marker", {_MARKER: True}), + _Case("response-cache", {_MARKER: "", "usage_object": {"cache_read_input_tokens": 100}}, "True"), + _Case("outside-before", {_MARKER: ""}, start_time=datetime(2026, 8, 31, 23, 59, 59)), + _Case( + "outside-after", {"usage_object": {"cache_read_input_tokens": 100}}, start_time=datetime(2026, 9, 2, 0, 0, 1) + ), +) + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cache_prisma( + _cache_postgresql: psycopg.Connection[tuple[object, ...]], +) -> AsyncIterator[Prisma]: + info: Final = _cache_postgresql.info + database: Final = Prisma(datasource={ + "url": f"postgresql://{info.user}@{info.host}:{info.port}/{info.dbname}?connection_limit=1", + }) + await database.connect() + try: + yield database + finally: + await database.disconnect() + + +def _seed(connection: psycopg.Connection[tuple[object, ...]], cases: tuple[_Case, ...] = _CASES) -> None: + with connection.cursor() as cursor: + cursor.execute(_DDL) + cursor.executemany( + """INSERT INTO "LiteLLM_SpendLogs" + VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s)""", + tuple( + ( + case.request_id, + case.start_time, + datetime(2026, 9, 1, 12, 0, 1), + _MODEL, + "dep-a", + "anthropic", + 0.01, + json.dumps(dict(case.metadata)), + case.cache_hit, + ) + for case in cases + ), + ) + connection.commit() + + +def _app(role: LitellmUserRoles | None) -> FastAPI: + application: Final = FastAPI() + application.include_router(router) + + def caller() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_role=role) + + application.dependency_overrides[user_api_key_auth] = caller + return application + + +@pytest.mark.asyncio +@pytest.mark.parametrize("filter", ["all", "injected", "hits"]) +@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) +async def test_request_filters_match_accounting_and_paginate_before_projection( + _cache_postgresql: psycopg.Connection[tuple[object, ...]], + _cache_prisma: Prisma, + monkeypatch: pytest.MonkeyPatch, + filter: PromptCachingRequestFilter, + role: LitellmUserRoles, +) -> None: + from litellm.proxy import proxy_server + + _seed(_cache_postgresql) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=_cache_prisma)) + monkeypatch.setattr(proxy_server, "llm_router", None) + expected: Final = tuple(sorted((case.request_id for case in _CASES if case.matches(filter)), reverse=True)) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=_app(role)), base_url="http://test") as client: + first: Final = await client.get( + _URL, params={"start_date": _START, "end_date": _END, "filter": filter, "page_size": 2} + ) + assert first.status_code == 200 + first_page: Final = PromptCachingRequestsResponse.model_validate_json(first.content) + assert tuple(row.request_id for row in first_page.requests) == expected[:2] + assert first_page.has_more is (len(expected) > 2) + assert (first_page.next_cursor is not None) is first_page.has_more + if first_page.next_cursor is not None: + assert first_page.next_cursor.request_id == expected[1] + assert first_page.next_cursor.start_time == first_page.requests[-1].start_time + next_response: Final = await client.get( + _URL, params={ + "start_date": _START, "end_date": _END, "filter": filter, "page_size": 2, + "cursor_start_time": first_page.next_cursor.start_time.astimezone( + timezone(timedelta(hours=-7)) + ).isoformat(), + "cursor_request_id": first_page.next_cursor.request_id, + } + ) + assert next_response.status_code == 200 + next_page: Final = PromptCachingRequestsResponse.model_validate_json(next_response.content) + assert tuple(row.request_id for row in next_page.requests) == expected[2:4] + assert next_page.has_more is (len(expected) > 4) + assert (next_page.next_cursor is not None) is next_page.has_more + second: Final = await client.get( + _URL, params={"start_date": _START, "end_date": _END, "filter": filter, "page_size": 100} + ) + assert second.status_code == 200 + complete: Final = PromptCachingRequestsResponse.model_validate_json(second.content) + assert tuple(row.request_id for row in complete.requests) == expected + assert complete.has_more is False + assert complete.next_cursor is None + assert all(row.start_time.tzinfo == timezone.utc for row in complete.requests) + payload: Final = _JSON_OBJECT.validate_json(second.content) + assert set(payload) == {"requests", "page_size", "has_more", "next_cursor"} + serialized_rows: Final = _JSON_ROWS.validate_python(payload["requests"]) + assert set(serialized_rows[0]) == { + "request_id", + "start_time", + "model", + "gateway_injected", + "cache_read_tokens", + "cache_creation_tokens", + "spend", + "net_savings", + } + by_id: Final = {row.request_id: row for row in complete.requests} + if filter == "all": + assert by_id["injected-empty"].gateway_injected is True + assert by_id["injected-empty"].net_savings is None + assert by_id["legacy-read"].gateway_injected is False + assert by_id["legacy-read"].net_savings is not None and by_id["legacy-read"].net_savings > 0 + assert by_id["write"].net_savings is not None and by_id["write"].net_savings < 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role", [None, LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +async def test_non_admin_is_denied_before_database_access( + role: LitellmUserRoles | None, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "prisma_client", None) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=_app(role)), base_url="http://test") as client: + response: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END}) + assert response.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("params", [ + {"filter": "savings"}, {"page_size": 0}, {"page_size": 101}, {"start_date": "invalid"}, + {"cursor_start_time": "invalid", "cursor_request_id": "request"}, + {"cursor_start_time": _START, "cursor_request_id": ""}, +]) +async def test_invalid_request_is_rejected(params: Mapping[str, str | int]) -> None: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=_app(LitellmUserRoles.PROXY_ADMIN)), base_url="http://test" + ) as client: + response: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END, **params}) + assert response.status_code == 422 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("params", [{"cursor_start_time": _START}, {"cursor_request_id": "request"}]) +async def test_incomplete_cursor_is_rejected( + params: Mapping[str, str], monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "prisma_client", None) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=_app(LitellmUserRoles.PROXY_ADMIN)), base_url="http://test" + ) as client: + response: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END, **params}) + assert response.status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("delete_before_cursor", [False, True]) +async def test_cursor_keeps_remaining_requests_once_during_insertions_and_deletions( + _cache_postgresql: psycopg.Connection[tuple[object, ...]], + _cache_prisma: Prisma, + monkeypatch: pytest.MonkeyPatch, + delete_before_cursor: bool, +) -> None: + from litellm.proxy import proxy_server + + cases: Final = (*_CASES, _Case( + "older-cache-read", {"usage_object": {"cache_read_input_tokens": 100}}, start_time=datetime(2026, 9, 1, 11), + )) + _seed(_cache_postgresql, cases) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=_cache_prisma)) + monkeypatch.setattr(proxy_server, "llm_router", None) + expected: Final = (*sorted((case.request_id for case in _CASES if case.matches("all")), reverse=True), "older-cache-read") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=_app(LitellmUserRoles.PROXY_ADMIN)), base_url="http://test" + ) as client: + first: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END, "page_size": 2}) + assert first.status_code == 200 + first_page: Final = PromptCachingRequestsResponse.model_validate_json(first.content) + assert tuple(row.request_id for row in first_page.requests) == expected[:2] + assert first_page.next_cursor is not None + with _cache_postgresql.cursor() as cursor: + cursor.executemany( + """INSERT INTO "LiteLLM_SpendLogs" + SELECT %s, %s, "endTime", model, model_id, custom_llm_provider, spend, metadata, cache_hit + FROM "LiteLLM_SpendLogs" WHERE request_id = %s""", + ( + ("newer-request", datetime(2026, 9, 1, 13), expected[0]), + ("zz-higher-id", cases[0].start_time, expected[0]), + ), + ) + if delete_before_cursor: + cursor.execute('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (expected[0],)) + _cache_postgresql.commit() + following: Final = await client.get(_URL, params={ + "start_date": _START, "end_date": _END, "page_size": 100, + "cursor_start_time": first_page.next_cursor.start_time.isoformat(), + "cursor_request_id": first_page.next_cursor.request_id, + }) + assert following.status_code == 200 + following_page: Final = PromptCachingRequestsResponse.model_validate_json(following.content) + assert tuple(row.request_id for row in following_page.requests) == expected[2:] + assert following_page.has_more is False + assert following_page.next_cursor is None diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index aae966022e3..004f07da431 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -11,6 +11,7 @@ from litellm.proxy.spend_tracking.savings import ( compute_autorouter_savings, compute_savings_spend, marks_gateway_injection, + prompt_caching_savings_for_request, ) from litellm.router import Router from litellm.types.utils import Usage @@ -18,6 +19,42 @@ from litellm.types.utils import Usage pytestmark = pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model,usage", [ + (None, {"cache_read_input_tokens": 100}), + ("claude-sonnet-5", None), + ("claude-sonnet-5", {"prompt_tokens": "invalid"}), +]) +def test_prompt_cache_estimate_distinguishes_unknown_from_zero(model: str | None, usage: dict[str, object] | None) -> None: + assert prompt_caching_savings_for_request(model, "anthropic", usage) is None + assert compute_savings_spend(model, "anthropic", 0, False, usage_object=usage).prompt_caching == 0 + assert prompt_caching_savings_for_request("claude-sonnet-5", "anthropic", {"prompt_tokens": 100}) == 0 + + +def test_prompt_cache_estimate_uses_the_rollup_pricing_and_retains_write_premiums() -> None: + router: Final = Router(model_list=[{ + "model_name": "negotiated", + "litellm_params": { + "model": "anthropic/claude-sonnet-5", "input_cost_per_token": 1e-6, + "cache_creation_input_token_cost": 1.25e-6, "cache_read_input_token_cost": 1e-7, + }, + "model_info": {"id": "negotiated-cache-prices"}, + }]) + + def current_router() -> Router: + return router + + usage: Final = {"cache_read_input_tokens": 1000, "cache_creation_input_tokens": 20000} + estimate: Final = prompt_caching_savings_for_request( + "claude-sonnet-5", "anthropic", usage, model_id="negotiated-cache-prices", llm_router=current_router, + ) + rollup: Final = compute_savings_spend( + "claude-sonnet-5", "anthropic", 0, True, usage_object=usage, + model_id="negotiated-cache-prices", llm_router=current_router, + ) + assert estimate == pytest.approx(1000 * (1e-6 - 1e-7) - 20000 * (1.25e-6 - 1e-6)) + assert estimate == rollup.prompt_caching == rollup.gateway_injected_caching + + @pytest.mark.parametrize("modifier", [{"speed": "fast"}, {"inference_geo": "us"}]) @pytest.mark.parametrize("continuing", [False, True]) def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], continuing: bool) -> None: diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index a0877b04648..f5b71a00061 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -3,7 +3,6 @@ import React, { useMemo, useState } from "react"; import { ArrowDown, ArrowUp, ArrowUpDown, Info } from "lucide-react"; -import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; @@ -81,7 +80,7 @@ const SortableHead = ({ }; const CacheLeakageCard: React.FC = ({ activity }) => { - const { dateValue, onDateChange, results, loading, isFetchingMore, apiKeyTruncation } = activity; + const { results, loading, isFetchingMore, apiKeyTruncation } = activity; const [dimension, setDimension] = useState("key"); const [sort, setSort] = useState({ column: "potentialSavings", dir: "desc" }); const leakage = useMemo(() => computeCacheLeakage(results, dimension), [results, dimension]); @@ -111,9 +110,6 @@ const CacheLeakageCard: React.FC = ({ activity }) => { cached token, after cache-write premiums.

-
- -
setDimension(value === "model" ? "model" : "key")}> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx index 03250e3e53b..f8336f5ab56 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -42,6 +42,7 @@ vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => })); vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () =>
})); +vi.mock("./PromptCachingRequestsTable", () => ({ default: () =>
})); import CostOptimizationView from "./CostOptimizationView"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.integration.test.tsx new file mode 100644 index 00000000000..833a46ce16f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.integration.test.tsx @@ -0,0 +1,248 @@ +import { Profiler } from "react"; +import { act, fireEvent, renderWithProviders, screen, testQueryClient, waitFor, within } from "@/../tests/test-utils"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { components } from "@/lib/http/schema"; +import PromptCachingRequestsTable from "./PromptCachingRequestsTable"; +import type { DateRange } from "./useDailyActivityRange"; + +type CacheRequest = components["schemas"]["PromptCachingRequest"]; +type RequestsResponse = components["schemas"]["PromptCachingRequestsResponse"]; +const firstCursor = { start_time: "2026-09-01T11:59:59.123456Z", request_id: "first-boundary?&" }; +const secondCursor = { start_time: firstCursor.start_time, request_id: "second-boundary" }; +const fetchMock = vi.fn(); +const dates = { from: new Date(2026, 8, 1, 12), to: new Date(2026, 8, 2, 12) }; +const request = (overrides: Partial = {}): CacheRequest => ({ + request_id: "request-default", + start_time: "2026-09-01T12:00:00Z", + model: "cache-test-model", + gateway_injected: true, + cache_read_tokens: 0, + cache_creation_tokens: 1000, + spend: 0.0375, + net_savings: -0.0075, + ...overrides, +}); +const response = (requests: CacheRequest[], nextCursor: RequestsResponse["next_cursor"] = null) => { + const body: RequestsResponse = { requests, has_more: nextCursor !== null, next_cursor: nextCursor, page_size: 50 }; + return Response.json(body); +}; +const lastQuery = () => new URL(String(fetchMock.mock.calls.at(-1)?.[0]), "http://localhost").searchParams; + +describe("PromptCachingRequestsTable", () => { + beforeEach(() => { + fetchMock.mockReset(); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + testQueryClient.clear(); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.useRealTimers(); + }); + + it("separates recorded injection from cache hits, retains write premiums and unknown savings, and links each request", async () => { + const clientHit = { + request_id: "client-hit", + gateway_injected: false, + cache_read_tokens: 10000, + cache_creation_tokens: 0, + net_savings: 0.27, + }; + fetchMock.mockResolvedValue( + response([ + request({ request_id: "injected/write?&", net_savings: -0.0075 }), + request(clientHit), + request({ request_id: "unknown-price", net_savings: null }), + request({ request_id: "no-benefit", net_savings: 0 }), + ]), + ); + renderWithProviders(); + + const table = await screen.findByRole("table", { name: "Prompt caching requests" }); + const write = within(table).getByRole("row", { name: /injected\/write/ }); + expect(within(write).getByText("Recorded")).toBeInTheDocument(); + expect(within(write).getByText("1,000")).toBeInTheDocument(); + expect(within(write).getByText("$0.0375")).toBeInTheDocument(); + expect(within(write).getByText("-$0.0075")).toBeInTheDocument(); + expect(within(write).getByText(new Date("2026-09-01T12:00:00Z").toLocaleString())).toBeInTheDocument(); + expect(within(write).getByText("cache-test-model")).toHaveAttribute("title", "cache-test-model"); + expect(within(write).getByRole("link")).toHaveAttribute("href", "/ui/logs?log_id=injected%2Fwrite%3F%26"); + + const hit = within(table).getByRole("row", { name: /client-hit/ }); + expect(within(hit).getByText("Not recorded")).toBeInTheDocument(); + expect(within(hit).getByText("10,000")).toBeInTheDocument(); + expect(within(hit).getByText("$0.2700")).toBeInTheDocument(); + expect(within(table).getByRole("row", { name: /unknown-price/ })).toHaveTextContent("Unavailable"); + expect(within(table).getByRole("row", { name: /no-benefit/ })).toHaveTextContent("$0.00"); + expect(screen.getByText(/after cache-write premiums/)).toBeInTheDocument(); + expect(lastQuery().get("start_date")).toBe("2026-09-01T00:00:00.000Z"); + expect(lastQuery().get("end_date")).toBe("2026-09-02T23:59:59.999Z"); + expect(fetchMock.mock.calls[0][1]?.headers).toEqual(expect.objectContaining({ Authorization: "Bearer token-a" })); + }); + + it("forwards complete server cursors, goes back to prior cursors, and clears them for each caching filter", async () => { + fetchMock.mockImplementation(async (input) => { + const query = new URL(String(input), "http://localhost").searchParams; + const pages = new Map([ + [null, 1], + [firstCursor.request_id, 2], + [secondCursor.request_id, 3], + ]); + const page = pages.get(query.get("cursor_request_id")); + const nextCursor = + new Map([ + [1, firstCursor], + [2, secondCursor], + ]).get(page ?? 0) ?? null; + return response([request({ request_id: `${query.get("filter")}-${page}` })], nextCursor); + }); + renderWithProviders(); + await screen.findByRole("link", { name: "all-1" }); + expect(screen.getByRole("button", { name: "Previous" })).toBeDisabled(); + expect(lastQuery().has("page")).toBe(false); + expect(lastQuery().has("cursor_request_id")).toBe(false); + + fireEvent.click(screen.getByRole("button", { name: "Next" })); + await screen.findByRole("link", { name: "all-2" }); + expect(screen.getByText("Page 2")).toBeInTheDocument(); + expect(lastQuery().get("cursor_start_time")).toBe(firstCursor.start_time); + expect(lastQuery().get("cursor_request_id")).toBe(firstCursor.request_id); + fireEvent.click(screen.getByRole("button", { name: "Next" })); + await screen.findByRole("link", { name: "all-3" }); + expect(screen.getByText("Page 3")).toBeInTheDocument(); + expect(lastQuery().get("cursor_start_time")).toBe(secondCursor.start_time); + expect(lastQuery().get("cursor_request_id")).toBe(secondCursor.request_id); + expect(screen.getByRole("button", { name: "Next" })).toBeDisabled(); + + await testQueryClient.invalidateQueries({ refetchType: "none" }); + fireEvent.click(screen.getByRole("button", { name: "Previous" })); + await screen.findByRole("link", { name: "all-2" }); + await waitFor(() => expect(lastQuery().get("cursor_request_id")).toBe(firstCursor.request_id)); + expect(lastQuery().get("cursor_start_time")).toBe(firstCursor.start_time); + expect(screen.getByText("Page 2")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Previous" })); + await screen.findByRole("link", { name: "all-1" }); + await waitFor(() => expect(lastQuery().has("cursor_request_id")).toBe(false)); + expect(lastQuery().has("cursor_start_time")).toBe(false); + fireEvent.click(screen.getByRole("button", { name: "Next" })); + await screen.findByRole("link", { name: "all-2" }); + + fireEvent.click(screen.getByRole("tab", { name: "LiteLLM injected" })); + await screen.findByRole("link", { name: "injected-1" }); + expect(screen.queryByRole("link", { name: "all-2" })).not.toBeInTheDocument(); + expect(lastQuery().get("filter")).toBe("injected"); + expect(lastQuery().has("cursor_request_id")).toBe(false); + expect(lastQuery().has("cursor_start_time")).toBe(false); + + fireEvent.click(screen.getByRole("button", { name: "Next" })); + await screen.findByRole("link", { name: "injected-2" }); + fireEvent.click(screen.getByRole("tab", { name: "Cache hits" })); + await screen.findByRole("link", { name: "hits-1" }); + expect(lastQuery().get("filter")).toBe("hits"); + expect(lastQuery().get("page_size")).toBe("50"); + expect(screen.getByText("Page 1")).toBeInTheDocument(); + }); + + it("includes the current UTC day for a range ending today, matching the activity totals", async () => { + vi.stubEnv("TZ", "America/Los_Angeles"); + vi.setSystemTime(new Date("2026-09-20T03:00:00Z")); + fetchMock.mockResolvedValue(response([])); + const today = { from: new Date(2026, 8, 19), to: new Date() }; + renderWithProviders(); + + await screen.findByText("No matching prompt caching requests in this range"); + expect(lastQuery().get("start_date")).toBe("2026-09-19T00:00:00.000Z"); + expect(lastQuery().get("end_date")).toBe("2026-09-20T23:59:59.999Z"); + }); + + it.each(["date", "authentication"])( + "hides every old-scope frame and resets pagination when %s changes", + async (change) => { + fetchMock.mockResolvedValueOnce(response([request({ request_id: "old-first" })], firstCursor)); + fetchMock.mockResolvedValueOnce(response([request({ request_id: "old-second" })])); + const committedOldRows: boolean[] = []; + const snapshot = () => { + committedOldRows.push(screen.queryByRole("link", { name: "old-second" }) !== null); + }; + const tree = (accessToken: string, dateValue: DateRange) => ( + + + + ); + const { rerender } = renderWithProviders(tree("token-a", dates)); + await screen.findByRole("link", { name: "old-first" }); + fireEvent.click(screen.getByRole("button", { name: "Next" })); + await screen.findByRole("link", { name: "old-second" }); + + const pending = Promise.withResolvers(); + fetchMock.mockReturnValueOnce(pending.promise); + committedOldRows.length = 0; + rerender( + tree( + change === "authentication" ? "token-b" : "token-a", + change === "date" ? { ...dates, to: new Date(2026, 8, 3) } : dates, + ), + ); + + expect(screen.getByRole("status")).toHaveTextContent("Loading requests"); + expect(committedOldRows.length).toBeGreaterThan(0); + expect(committedOldRows.every((visible) => !visible)).toBe(true); + expect(lastQuery().has("cursor_request_id")).toBe(false); + expect(lastQuery().has("cursor_start_time")).toBe(false); + if (change === "date") { + expect(lastQuery().get("end_date")).toBe("2026-09-03T23:59:59.999Z"); + } else { + expect(fetchMock.mock.calls.at(-1)?.[1]?.headers).toEqual( + expect.objectContaining({ Authorization: "Bearer token-b" }), + ); + } + + pending.resolve(response([request({ request_id: "new-first" })])); + await screen.findByRole("link", { name: "new-first" }); + expect(screen.getByText("Page 1")).toBeInTheDocument(); + expect(committedOldRows.every((visible) => !visible)).toBe(true); + }, + ); + + it("ignores a delayed response from the previous caching filter", async () => { + const stale = Promise.withResolvers(); + const current = Promise.withResolvers(); + fetchMock.mockReturnValueOnce(stale.promise).mockReturnValueOnce(current.promise); + renderWithProviders(); + fireEvent.click(screen.getByRole("tab", { name: "Cache hits" })); + expect(lastQuery().get("filter")).toBe("hits"); + + current.resolve(response([request({ request_id: "current-hit" })])); + await screen.findByRole("link", { name: "current-hit" }); + await act(async () => { + stale.resolve(response([request({ request_id: "stale-all" })], firstCursor)); + await stale.promise; + }); + + expect(screen.getByRole("link", { name: "current-hit" })).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "stale-all" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Next" })).toBeDisabled(); + }); + + it("offers retry after a failed read and shows the empty state after it succeeds", async () => { + fetchMock.mockRejectedValueOnce(new Error("offline")); + fetchMock.mockResolvedValueOnce(response([])); + renderWithProviders(); + + expect(await screen.findByRole("alert")).toHaveTextContent("Could not load prompt caching requests"); + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByText("No matching prompt caching requests in this range")).toBeInTheDocument(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Next" })).toBeDisabled(); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("does not request data for an incomplete date range", async () => { + renderWithProviders(); + expect(screen.getByText("Select a date range to view requests")).toBeInTheDocument(); + expect(screen.queryByRole("status")).not.toBeInTheDocument(); + await waitFor(() => expect(fetchMock).not.toHaveBeenCalled()); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.tsx new file mode 100644 index 00000000000..29aa9252e7b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingRequestsTable.tsx @@ -0,0 +1,186 @@ +"use client"; + +import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; +import Link from "next/link"; +import { useState } from "react"; + +import { apiClient } from "@/components/networking"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { LOG_ID_QUERY_PARAM } from "@/components/view_logs/logDetailRouting"; +import type { paths } from "@/lib/http/schema"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { uiHref } from "@/utils/uiHref"; +import { usd } from "./costOptimizationUtils"; +import { benchmarksWindow as activityWindow } from "./useAutoRouterBenchmarks"; +import type { DateRange } from "./useDailyActivityRange"; + +const REQUESTS_PATH = "/cost_optimization/prompt_caching/requests"; +type RequestsEndpoint = paths[typeof REQUESTS_PATH]["get"]; +type RequestsResponse = RequestsEndpoint["responses"][200]["content"]["application/json"]; +type RequestsQuery = NonNullable; +type RequestFilter = NonNullable; +type RequestCursor = RequestsResponse["next_cursor"]; + +interface PromptCachingRequestsTableProps { + accessToken: string; + dateValue: DateRange; +} + +export default function PromptCachingRequestsTable({ accessToken, dateValue }: PromptCachingRequestsTableProps) { + const [filter, setFilter] = useState("all"); + const window = activityWindow(dateValue, new Date()); + const startDate = window.start_date ? `${window.start_date}T00:00:00.000Z` : ""; + const endDate = window.end_date ? `${window.end_date}T23:59:59.999Z` : ""; + const scope = JSON.stringify([accessToken, startDate, endDate, filter]); + const [pagination, setPagination] = useState<{ scope: string; cursors: readonly RequestCursor[] }>({ + scope, + cursors: [null], + }); + const cursors = pagination.scope === scope ? pagination.cursors : [null]; + const cursor = cursors.at(-1); + const page = cursors.length; + + if (pagination.scope !== scope) { + setPagination({ scope, cursors: [null] }); + } + + const enabled = Boolean(accessToken && startDate && endDate); + const query: RequestsQuery = { + start_date: startDate, + end_date: endDate, + filter, + page_size: 50, + cursor_start_time: cursor?.start_time, + cursor_request_id: cursor?.request_id, + }; + const queryOptions: UseQueryOptions = { + queryKey: [REQUESTS_PATH, accessToken, query], + queryFn: ({ signal }) => apiClient.get(REQUESTS_PATH, { accessToken, query, signal }), + enabled, + retry: false, + }; + const requests = useQuery(queryOptions); + const nextCursor = requests.data?.next_cursor; + + const changeFilter = (value: unknown) => { + if (value === "all" || value === "injected" || value === "hits") { + setFilter(value); + } + }; + + return ( + + +
+ Prompt caching requests +

+ Requests with recorded LiteLLM injection or provider cache reads or writes. A cache hit alone does not + establish LiteLLM injection; older logs may not record it. +

+

+ Net savings are estimated from logged usage and current configured pricing, after cache-write premiums. + Negative values mean caching cost more; unavailable means the request could not be priced. +

+
+ + + All caching + LiteLLM injected + Cache hits + + + + + {!enabled &&

Select a date range to view requests

} + {enabled && requests.isPending && ( +

+ Loading requests... +

+ )} + {enabled && requests.isError && ( +
+

Could not load prompt caching requests

+ +
+ )} + {enabled && requests.isSuccess && ( + <> + {requests.data.requests.length === 0 ? ( +

+ No matching prompt caching requests in this range +

+ ) : ( + + + + Request + Model + LiteLLM injection + Cache reads + Cache writes + Actual cost + Net savings + + + + {requests.data.requests.map((request) => ( + + + + {request.request_id} + + + + + + {request.model} + + + {request.gateway_injected ? "Recorded" : "Not recorded"} + {formatNumberWithCommas(request.cache_read_tokens)} + + {formatNumberWithCommas(request.cache_creation_tokens)} + + {usd(request.spend)} + + {request.net_savings === null ? "Unavailable" : usd(request.net_savings)} + + + ))} + +
+ )} +
+ + Page {page} + +
+ + )} + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx index 66db347e70f..35464c5852e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx @@ -1,4 +1,4 @@ -import { render, waitFor, screen } from "@testing-library/react"; +import { fireEvent, render, waitFor, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; const mockGetGeneralSettingsCall = vi.fn(); @@ -12,6 +12,21 @@ vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => })); const mockCacheLeakageCard = vi.fn(); +const mockRequestsTable = vi.fn(); +const nextDateRange = { from: new Date(2026, 8, 1), to: new Date(2026, 8, 2) }; + +vi.mock("./PromptCachingRequestsTable", () => ({ + default: (props: unknown) => { + mockRequestsTable(props); + return
; + }, +})); + +vi.mock("@/components/shared/advanced_date_picker", () => ({ + default: ({ onValueChange }: { onValueChange: (range: typeof nextDateRange) => void }) => ( + + ), +})); vi.mock("./CacheLeakageCard", () => ({ __esModule: true, @@ -24,7 +39,7 @@ vi.mock("./CacheLeakageCard", () => ({ import PromptCachingTab from "./PromptCachingTab"; describe("PromptCachingTab", () => { - it("renders the cache leakage table alongside the caching settings", async () => { + it("shares the selected dates between requests and cache leakage alongside caching settings", async () => { mockGetGeneralSettingsCall.mockResolvedValue([]); const activity = { @@ -42,6 +57,10 @@ describe("PromptCachingTab", () => { expect(screen.getByTestId("caching-settings")).toBeInTheDocument(); expect(screen.getByTestId("cache-leakage-card")).toBeInTheDocument(); + expect(screen.getByTestId("caching-requests")).toBeInTheDocument(); + expect(mockRequestsTable).toHaveBeenCalledWith({ accessToken: "test-token", dateValue: activity.dateValue }); + fireEvent.click(screen.getByRole("button", { name: "Change caching dates" })); + expect(activity.onDateChange).toHaveBeenCalledWith(nextDateRange); await waitFor(() => expect(mockCacheLeakageCard).toHaveBeenCalledWith(expect.objectContaining({ activity }))); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx index 59b38f272e0..4e43317998e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx @@ -3,12 +3,14 @@ import React, { useCallback, useEffect, useState } from "react"; import { getGeneralSettingsCall } from "@/components/networking"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { toast } from "@/lib/toast"; import { PromptCachingPanel, generalSettingsItem, } from "@/app/(dashboard)/router-settings/_components/general_settings"; import CacheLeakageCard from "./CacheLeakageCard"; +import PromptCachingRequestsTable from "./PromptCachingRequestsTable"; import { DailyActivityRange } from "./useDailyActivityRange"; interface PromptCachingTabProps { @@ -48,6 +50,11 @@ const PromptCachingTab: React.FC = ({ accessToken, activi return (
+
+

Date range for requests and cache leakage

+ +
+
); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 81580c8bfb1..d916509c06f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -3534,6 +3534,23 @@ export interface paths { patch?: never; trace?: never; }; + "/cost_optimization/prompt_caching/requests": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Prompt Caching Requests */ + get: operations["get_prompt_caching_requests_cost_optimization_prompt_caching_requests_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/credentials": { parameters: { query?: never; @@ -35814,6 +35831,48 @@ export interface components { prompt_id: string; prompt_info?: components["schemas"]["PromptInfo"] | null; }; + /** PromptCachingRequest */ + PromptCachingRequest: { + /** Cache Creation Tokens */ + cache_creation_tokens: number; + /** Cache Read Tokens */ + cache_read_tokens: number; + /** Gateway Injected */ + gateway_injected: boolean; + /** Model */ + model: string; + /** Net Savings */ + net_savings: number | null; + /** Request Id */ + request_id: string; + /** Spend */ + spend: number; + /** + * Start Time + * Format: date-time + */ + start_time: string; + }; + /** PromptCachingRequestCursor */ + PromptCachingRequestCursor: { + /** Request Id */ + request_id: string; + /** + * Start Time + * Format: date-time + */ + start_time: string; + }; + /** PromptCachingRequestsResponse */ + PromptCachingRequestsResponse: { + /** Has More */ + has_more: boolean; + next_cursor: components["schemas"]["PromptCachingRequestCursor"] | null; + /** Page Size */ + page_size: number; + /** Requests */ + requests: components["schemas"]["PromptCachingRequest"][]; + }; /** PromptInfo */ PromptInfo: { /** @@ -47238,6 +47297,42 @@ export interface operations { }; }; }; + get_prompt_caching_requests_cost_optimization_prompt_caching_requests_get: { + parameters: { + query: { + start_date: string; + end_date: string; + page_size?: number; + filter?: "all" | "injected" | "hits"; + cursor_start_time?: string | null; + cursor_request_id?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PromptCachingRequestsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_credentials_credentials_get: { parameters: { query?: never;