From b1360efc2f5237b2aa5fd99c04c509e929e5e056 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:03:38 -0700 Subject: [PATCH] fix(proxy): attribute gate-rejected requests to their endpoint in cache analytics (#40824) * fix(proxy): attribute gate-rejected requests to their endpoint in cache analytics Requests rejected before dispatch (bad key, blocked key, budget, rate limit, malformed body) were spend-logged with an empty call_type because the synthesized logging object never reached the failure lifter. The caching dashboard rolled all of them, plus failed calls on info routes such as /model/info, into one Unknown group. Resolve call_type from the matched route first, falling back to body shape, and keep the synthesized logging object on request_data so the lifter sees it. Log bare auth exceptions with the 401 ProxyException the client gets so error_code is never empty. Exclude info routes from the cache analytics groups and error breakdown. The dashboard explains the Unknown group when older rows still produce one. Resolves LIT-5884 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep the raw auth exception for failure callbacks Record the client-facing status in the spend log through a separate client_exception argument so custom failure callbacks still receive the exception auth raised. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep the route for multi-operation endpoints and exclude info routes from cache filter options Routes such as /v1/files map to several operations (create, list) and the method is not available in the failure hook, so a rejected request there is filed under its route instead of the first mapped call type. The key alias and model filter-option queries now apply the same info-route exclusion as the groups and error breakdown, so every offered filter value returns data. The info-route exclusion and Unknown grouping are now covered against a real Postgres in tests/proxy_behavior/spend/test_cache_activity.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): drop client_exception, the spend log row never used it The DB spend row for a gate rejection is written by _ProxyDBLogger from the original exception, so the status-bearing copy only reached the in-memory logging payload. Live runs at the tip still recorded bare auth exceptions as Unknown/Exception, the same as the base branch. Removing the plumbing keeps this PR to endpoint attribution and the info-route exclusion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../analytics_endpoints/cache_activity.py | 17 ++- litellm/proxy/utils.py | 36 ++++--- .../spend/test_cache_activity.py | 102 ++++++++++++++++++ .../test_analytics_endpoints.py | 18 ++++ .../test_post_call_failure_hook.py | 83 +++++++++++--- .../_components/cache_dashboard.test.tsx | 29 +++++ .../caching/_components/cache_dashboard.tsx | 7 ++ 7 files changed, 258 insertions(+), 34 deletions(-) create mode 100644 tests/proxy_behavior/spend/test_cache_activity.py diff --git a/litellm/proxy/analytics_endpoints/cache_activity.py b/litellm/proxy/analytics_endpoints/cache_activity.py index b87b8eac3ef..902e3fb3db3 100644 --- a/litellm/proxy/analytics_endpoints/cache_activity.py +++ b/litellm/proxy/analytics_endpoints/cache_activity.py @@ -6,10 +6,13 @@ from typing import TYPE_CHECKING, Final from pydantic import BaseModel, TypeAdapter +from litellm.proxy._types import LiteLLMRoutes + if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient UNKNOWN_CALL_TYPE: Final = "Unknown" +INFO_ROUTES_JSON: Final = json.dumps(LiteLLMRoutes.info_routes.value) class CacheActivityGroup(BaseModel): @@ -69,6 +72,7 @@ GROUPS_SQL: Final = """ OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb))) AND ($4::jsonb = '[]'::jsonb OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb))) + AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb)) GROUP BY 1 ORDER BY (COUNT(*)) DESC """ @@ -89,6 +93,7 @@ ERROR_BREAKDOWN_SQL: Final = """ OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb))) AND ($4::jsonb = '[]'::jsonb OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb))) + AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb)) GROUP BY 1, 2, 3 ORDER BY (COUNT(*)) DESC """ @@ -100,6 +105,7 @@ KEY_ALIAS_OPTIONS_SQL: Final = """ WHERE sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') + AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb)) ORDER BY 1 """ @@ -110,6 +116,7 @@ MODEL_OPTIONS_SQL: Final = """ sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') AND sl."model" != '' + AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb)) ORDER BY 1 """ @@ -152,10 +159,12 @@ async def get_cache_activity( key_aliases_json: Final = json.dumps(list(key_aliases)) models_json: Final = json.dumps(list(models)) group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather( - prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json), - prisma_client.db.query_raw(ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json), - prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date), - prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date), + prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON), + prisma_client.db.query_raw( + ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON + ), + prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON), + prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON), ) groups: Final = _groups_adapter.validate_python(group_rows or []) return CacheActivityResponse( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 89625021e37..c095586b6c9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -93,6 +93,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert +from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, get_or_create_metadata_bucket, @@ -879,6 +880,18 @@ def _failure_usage_to_lift( _EMPTY_LIFT: Final = MappingProxyType({}) +def _call_type_for_route(route: str | None) -> str | None: + """The route's call type when it maps to a single operation (its async and sync variants); + None for routes shared by several operations, since the method is not known here.""" + if route is None: + return None + call_types: Final = get_call_types_for_route(route) + if not call_types: + return None + operations: Final = frozenset(call_type.value.removeprefix("a") for call_type in call_types) + return call_types[0].value if len(operations) == 1 else None + + def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]: """Failure-path callbacks run after ``litellm_logging_obj`` is popped from request_data (it is not serialisable), so the caller merges these fields @@ -2549,10 +2562,6 @@ class ProxyLogging: @staticmethod def _stream_requires_guardrail_translation(user_api_key_dict: UserAPIKeyAuth) -> bool: - from litellm.litellm_core_utils.api_route_to_call_types import ( - get_call_types_for_route, - ) - route: Final = user_api_key_dict.request_route if not route: return False @@ -3020,6 +3029,7 @@ class ProxyLogging: start_time=datetime.now(), **request_data, ) + request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller if "metadata" not in request_data: request_data["metadata"] = {} request_data["metadata"].update(user_api_key_logged_metadata) @@ -3044,25 +3054,23 @@ class ProxyLogging: ) input: list | str | dict = "" - normalized_call_type: str | None = None + body_shape_call_type: str | None = None if "messages" in request_data and isinstance(request_data["messages"], list): input = request_data["messages"] litellm_logging_obj.model_call_details["messages"] = input - if litellm_logging_obj.call_type != CallTypes.pass_through.value: - normalized_call_type = CallTypes.acompletion.value + body_shape_call_type = CallTypes.acompletion.value elif "prompt" in request_data and isinstance(request_data["prompt"], str): input = request_data["prompt"] litellm_logging_obj.model_call_details["prompt"] = input - if litellm_logging_obj.call_type != CallTypes.pass_through.value: - normalized_call_type = CallTypes.atext_completion.value + body_shape_call_type = CallTypes.atext_completion.value elif "input" in request_data and isinstance(request_data["input"], list): input = request_data["input"] litellm_logging_obj.model_call_details["input"] = input - if litellm_logging_obj.call_type != CallTypes.pass_through.value: - normalized_call_type = CallTypes.aembedding.value - if normalized_call_type is not None: - litellm_logging_obj.call_type = normalized_call_type - litellm_logging_obj.model_call_details["call_type"] = normalized_call_type + body_shape_call_type = CallTypes.aembedding.value + resolved_call_type: Final = _call_type_for_route(route) or body_shape_call_type + if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value: + litellm_logging_obj.call_type = resolved_call_type + litellm_logging_obj.model_call_details["call_type"] = resolved_call_type # Pass-through endpoints are logged via the callback loop's # async_post_call_failure_hook — skip pre_call and failure handlers. if litellm_logging_obj.call_type == CallTypes.pass_through.value: diff --git a/tests/proxy_behavior/spend/test_cache_activity.py b/tests/proxy_behavior/spend/test_cache_activity.py new file mode 100644 index 00000000000..f4a7e8eb2b2 --- /dev/null +++ b/tests/proxy_behavior/spend/test_cache_activity.py @@ -0,0 +1,102 @@ +""" +Behavior tests for the cache analytics queries against a real Postgres. The info-route +exclusion and the Unknown grouping live in SQL, so these tests are the ones that exercise +them; the endpoint wiring is unit-tested in +tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py. +""" + +import json +import uuid +from datetime import datetime +from typing import Final + +import pytest + +from litellm.proxy.analytics_endpoints.cache_activity import ( + ERROR_BREAKDOWN_SQL, + GROUPS_SQL, + INFO_ROUTES_JSON, + KEY_ALIAS_OPTIONS_SQL, + MODEL_OPTIONS_SQL, +) + +pytestmark = pytest.mark.asyncio(loop_scope="session") + +DAY: Final = datetime(2001, 3, 7) +AT_NOON: Final = DAY.replace(hour=12) +RUN: Final = uuid.uuid4() +INFERENCE_KEY: Final = f"ca-inference-{RUN}" +INFO_ONLY_KEY: Final = f"ca-info-only-{RUN}" +INFERENCE_ALIAS: Final = f"alias-inference-{RUN}" +INFO_ONLY_ALIAS: Final = f"alias-info-only-{RUN}" +INFERENCE_MODEL: Final = f"gpt-5.4-mini-{RUN}" +INFO_ONLY_MODEL: Final = f"ghost-model-{RUN}" +NO_FILTER: Final = "[]" + + +async def _spend_log(db, api_key: str, call_type: str, status: str, model: str = "", error_code: str = "") -> None: + metadata: Final = {"error_information": {"error_code": error_code, "error_class": "ProxyException"}} + await db.execute_raw( + 'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "api_key", "startTime", "endTime", "model", ' + '"status", "metadata") VALUES ($1, $2, $3, $4::timestamp, $4::timestamp, $5, $6, $7::jsonb)', + str(uuid.uuid4()), + call_type, + api_key, + AT_NOON, + model, + status, + json.dumps(metadata if status == "failure" else {}), + ) + + +@pytest.fixture(scope="module", autouse=True) +async def seeded(db): + for token, alias in ((INFERENCE_KEY, INFERENCE_ALIAS), (INFO_ONLY_KEY, INFO_ONLY_ALIAS)): + await db.execute_raw( + 'INSERT INTO "LiteLLM_VerificationToken" ("token", "key_alias") VALUES ($1, $2)', token, alias + ) + await _spend_log(db, INFERENCE_KEY, "acompletion", "success", model=INFERENCE_MODEL) + await _spend_log(db, INFERENCE_KEY, "acompletion", "failure", model=INFERENCE_MODEL, error_code="429") + await _spend_log(db, INFERENCE_KEY, "", "failure", error_code="401") + await _spend_log(db, INFERENCE_KEY, "/model/info", "failure", error_code="401") + await _spend_log(db, INFO_ONLY_KEY, "/v1/models", "failure", model=INFO_ONLY_MODEL, error_code="401") + await _spend_log(db, INFO_ONLY_KEY, "/key/info", "success") + yield + keys: Final = [INFERENCE_KEY, INFO_ONLY_KEY] + await db.execute_raw('DELETE FROM "LiteLLM_SpendLogs" WHERE "api_key" = ANY($1::text[])', keys) + await db.execute_raw('DELETE FROM "LiteLLM_VerificationToken" WHERE "token" = ANY($1::text[])', keys) + + +async def _groups(db, key_aliases: list[str]) -> dict[str, dict]: + rows: Final = await db.query_raw(GROUPS_SQL, DAY, DAY, json.dumps(key_aliases), NO_FILTER, INFO_ROUTES_JSON) + return {row["call_type"]: row for row in rows} + + +async def test_groups_drop_info_routes_and_keep_unknown_for_rows_without_an_endpoint(db): + groups: Final = await _groups(db, [INFERENCE_ALIAS]) + assert set(groups) == {"acompletion", "Unknown"} + assert (groups["acompletion"]["api_requests"], groups["acompletion"]["failed_requests"]) == (1, 1) + assert (groups["Unknown"]["api_requests"], groups["Unknown"]["failed_requests"]) == (0, 1) + + +async def test_key_with_only_info_route_traffic_has_no_groups(db): + assert await _groups(db, [INFO_ONLY_ALIAS]) == {} + + +async def test_error_breakdown_drops_info_routes(db): + rows: Final = await db.query_raw( + ERROR_BREAKDOWN_SQL, DAY, DAY, json.dumps([INFERENCE_ALIAS, INFO_ONLY_ALIAS]), NO_FILTER, INFO_ROUTES_JSON + ) + assert {(row["call_type"], row["error_code"], row["count"]) for row in rows} == { + ("acompletion", "429", 1), + ("Unknown", "401", 1), + } + + +async def test_filter_options_only_offer_values_that_return_analytics(db): + key_alias_rows: Final = await db.query_raw(KEY_ALIAS_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON) + model_rows: Final = await db.query_raw(MODEL_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON) + key_aliases: Final = {row["key_alias"] for row in key_alias_rows} + models: Final = {row["model"] for row in model_rows} + assert INFERENCE_ALIAS in key_aliases and INFO_ONLY_ALIAS not in key_aliases + assert INFERENCE_MODEL in models and INFO_ONLY_MODEL not in models diff --git a/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py b/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py index 4072f83511e..ea528878db8 100644 --- a/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py +++ b/tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py @@ -12,10 +12,13 @@ from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException +from litellm.proxy._types import LiteLLMRoutes from litellm.proxy.analytics_endpoints.analytics_endpoints import get_global_activity from litellm.proxy.analytics_endpoints.cache_activity import ( ERROR_BREAKDOWN_SQL, GROUPS_SQL, + KEY_ALIAS_OPTIONS_SQL, + MODEL_OPTIONS_SQL, CacheActivityGroup, compute_totals, ) @@ -112,6 +115,21 @@ async def test_filters_are_passed_to_sql_as_json_arrays(mock_prisma: MagicMock): assert call.args[4] == json.dumps(["gpt-5.1", "claude-opus-4-8"]) +@pytest.mark.asyncio +async def test_every_query_excludes_the_same_info_routes(mock_prisma: MagicMock): + """Regression for LIT-5884: failed info-route calls are spend-logged but are not inference traffic, so + the groups, error breakdown and both filter-option queries all receive the same exclusion list. What + the SQL does with it is covered against Postgres in tests/proxy_behavior/spend/test_cache_activity.py.""" + await get_global_activity(start_date="2026-07-01", end_date="2026-07-27", key_aliases=[], models=[]) + + exclusions_by_query = {call.args[0]: json.loads(call.args[-1]) for call in mock_prisma.db.query_raw.call_args_list} + assert set(exclusions_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL} + for excluded_call_types in exclusions_by_query.values(): + assert excluded_call_types == LiteLLMRoutes.info_routes.value + assert {"/model/info", "/v1/models", "/key/info"} <= set(excluded_call_types) + assert "" not in excluded_call_types + + @pytest.mark.asyncio async def test_rejects_malformed_dates_with_400(mock_prisma: MagicMock): with pytest.raises(HTTPException) as exc_info: diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py index a2a57931d26..8b2b6b4c6ca 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -33,9 +33,7 @@ def test_is_proxy_only_llm_api_truth_table(proxy_logging): snapshot. Covers no-route, non-LLM route, HTTPException on LLM route, and auth-error short-circuit.""" snapshot = { - "no_route": proxy_logging._is_proxy_only_llm_api_error( - original_exception=Exception(), route=None - ), + "no_route": proxy_logging._is_proxy_only_llm_api_error(original_exception=Exception(), route=None), "non_llm_route": proxy_logging._is_proxy_only_llm_api_error( original_exception=HTTPException(status_code=429, detail="rate"), route="/random/path", @@ -158,9 +156,7 @@ async def test_post_call_failure_hook_non_http_exception_in_callback_swallowed( @pytest.mark.asyncio -async def test_handle_logging_proxy_only_path_uses_existing_logging_obj( - proxy_logging, make_user_api_key_auth -): +async def test_handle_logging_proxy_only_path_uses_existing_logging_obj(proxy_logging, make_user_api_key_auth): logging_obj = MagicMock() logging_obj.call_type = "acompletion" logging_obj.model_call_details = {} @@ -183,10 +179,7 @@ async def test_handle_logging_proxy_only_path_uses_existing_logging_obj( snapshot = { "input_logged": "messages" in logging_obj.model_call_details, "call_type_normalized": logging_obj.call_type, - "marker_present": logging_obj.model_call_details.get( - LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL - ) - is True, + "marker_present": logging_obj.model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL) is True, "async_failure_called": logging_obj.async_failure_handler.called, } assert snapshot == { @@ -198,9 +191,7 @@ async def test_handle_logging_proxy_only_path_uses_existing_logging_obj( @pytest.mark.asyncio -async def test_handle_logging_proxy_only_path_skips_for_pass_through( - proxy_logging, make_user_api_key_auth -): +async def test_handle_logging_proxy_only_path_skips_for_pass_through(proxy_logging, make_user_api_key_auth): from litellm.types.utils import CallTypes logging_obj = MagicMock() @@ -248,9 +239,7 @@ async def test_handle_logging_proxy_only_path_no_logging_obj_creates_one( @pytest.mark.asyncio -async def test_handle_logging_proxy_only_path_propagates_async_failure_raises( - proxy_logging, make_user_api_key_auth -): +async def test_handle_logging_proxy_only_path_propagates_async_failure_raises(proxy_logging, make_user_api_key_auth): logging_obj = MagicMock() logging_obj.call_type = "acompletion" logging_obj.model_call_details = {} @@ -267,3 +256,65 @@ async def test_handle_logging_proxy_only_path_propagates_async_failure_raises( route="/chat/completions", original_exception=Exception("x"), ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route, request_data, expected_call_type", + [ + ("/v1/chat/completions", {}, "acompletion"), + ("/chat/completions", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}, "acompletion"), + ("/v1/messages", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}, "anthropic_messages"), + ("/v1/responses", {"model": "m", "input": "hi"}, "aresponses"), + ("/v1/embeddings", {"model": "m", "input": ["hi"]}, "aembedding"), + ("/model/info", {}, "/model/info"), + ], +) +async def test_post_call_failure_hook_lifts_route_call_type_for_gate_rejections( + proxy_logging, make_user_api_key_auth, route, request_data, expected_call_type +): + """Regression for LIT-5884: the matched route, not the body shape, sets the + spend-log call_type for requests rejected before dispatch.""" + proxy_logging.alert_types = [] + await proxy_logging.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("Authentication Error, No api key passed in."), + user_api_key_dict=make_user_api_key_auth(request_route=route), + error_type=ProxyErrorTypes.auth_error, + route=route, + ) + assert request_data["call_type"] == expected_call_type + assert "start_time" in request_data + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_falls_back_to_body_shape_without_a_route(proxy_logging, make_user_api_key_auth): + proxy_logging.alert_types = [] + request_data = {"model": "m", "messages": [{"role": "user", "content": "hi"}]} + await proxy_logging.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("Authentication Error, No api key passed in."), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + error_type=ProxyErrorTypes.auth_error, + ) + assert request_data["call_type"] == "acompletion" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/v1/files", "/files/file-abc", "/v1/containers"]) +async def test_post_call_failure_hook_keeps_the_route_for_multi_operation_routes( + proxy_logging, make_user_api_key_auth, route +): + """Routes shared by several operations (POST create vs GET list) cannot be attributed without the + method, so a rejected request there is filed under its route, not under whichever operation the + mapping lists first.""" + proxy_logging.alert_types = [] + request_data: dict = {} + await proxy_logging.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("Authentication Error, No api key passed in."), + user_api_key_dict=make_user_api_key_auth(request_route=route), + error_type=ProxyErrorTypes.auth_error, + route=route, + ) + assert request_data["call_type"] == route diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx index deb819fdc04..3447602dd79 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx @@ -236,6 +236,35 @@ describe("CacheDashboard cache analytics charts", () => { expect(screen.queryByText(/Failed requests by error code/)).not.toBeInTheDocument(); }); + it("explains the Unknown bucket only when a group has no recorded endpoint", async () => { + const { rerender } = renderDashboard(); + await screen.findByText(REQUESTS_CHART_TITLE); + expect(screen.queryByText(/recorded no endpoint/)).not.toBeInTheDocument(); + + useCacheActivity.mockReturnValue({ + data: { + ...cacheActivity, + groups: [ + ...cacheActivity.groups, + { + call_type: "Unknown", + api_requests: 0, + cache_hits: 0, + failed_requests: 121000, + cached_completion_tokens: 0, + generated_completion_tokens: 0, + }, + ], + }, + refetch: vi.fn(), + }); + rerender(); + + expect( + within(cardTitled(REQUESTS_CHART_TITLE)).getByText(/Unknown groups spend logs that recorded no endpoint/), + ).toHaveTextContent("not necessarily LLM API requests"); + }); + it("formats y-axis ticks with compact notation", async () => { renderDashboard(); const { requestsCard, tokensCard } = await findChartCards(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index befc0b3c5ca..29819adf524 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -35,6 +35,11 @@ const REQUEST_SERIES = { failed: "Failed requests", } as const; +const UNKNOWN_CALL_TYPE = "Unknown"; + +const UNKNOWN_CALL_TYPE_NOTE = + "Unknown groups spend logs that recorded no endpoint. Older proxy versions wrote those for requests rejected before routing, so they are not necessarily LLM API requests."; + const toChartDatum = (group: CacheActivityGroup) => ({ name: group.call_type, [REQUEST_SERIES.apiRequests]: group.api_requests, @@ -103,6 +108,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole const uniqueApiKeys = activity?.filter_options.key_aliases ?? []; const uniqueModels = activity?.filter_options.models ?? []; const chartData = (activity?.groups ?? []).map(toChartDatum); + const hasUnknownGroup = (activity?.groups ?? []).some((group) => group.call_type === UNKNOWN_CALL_TYPE); const activeDrilldownCallType = resolveDrilldownCallType(errorDrilldownCallType, activity?.groups ?? []); const handleRefreshClick = () => { @@ -288,6 +294,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole

Click a red failed-requests segment to see which error codes caused those failures.

+ {hasUnknownGroup &&

{UNKNOWN_CALL_TYPE_NOTE}

}