mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(proxy): attribute gate-rejected requests to their endpoint in cache analytics (#40824)
* fix(proxy): attribute gate-rejected requests to their endpoint in cache analytics Requests rejected before dispatch (bad key, blocked key, budget, rate limit, malformed body) were spend-logged with an empty call_type because the synthesized logging object never reached the failure lifter. The caching dashboard rolled all of them, plus failed calls on info routes such as /model/info, into one Unknown group. Resolve call_type from the matched route first, falling back to body shape, and keep the synthesized logging object on request_data so the lifter sees it. Log bare auth exceptions with the 401 ProxyException the client gets so error_code is never empty. Exclude info routes from the cache analytics groups and error breakdown. The dashboard explains the Unknown group when older rows still produce one. Resolves LIT-5884 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep the raw auth exception for failure callbacks Record the client-facing status in the spend log through a separate client_exception argument so custom failure callbacks still receive the exception auth raised. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep the route for multi-operation endpoints and exclude info routes from cache filter options Routes such as /v1/files map to several operations (create, list) and the method is not available in the failure hook, so a rejected request there is filed under its route instead of the first mapped call type. The key alias and model filter-option queries now apply the same info-route exclusion as the groups and error breakdown, so every offered filter value returns data. The info-route exclusion and Unknown grouping are now covered against a real Postgres in tests/proxy_behavior/spend/test_cache_activity.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): drop client_exception, the spend log row never used it The DB spend row for a gate rejection is written by _ProxyDBLogger from the original exception, so the status-bearing copy only reached the in-memory logging payload. Live runs at the tip still recorded bare auth exceptions as Unknown/Exception, the same as the base branch. Removing the plumbing keeps this PR to endpoint attribution and the info-route exclusion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng <yucheng@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
c134fb7a38
commit
b1360efc2f
7 changed files with 258 additions and 34 deletions
|
|
@ -6,10 +6,13 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm.proxy._types import LiteLLMRoutes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
UNKNOWN_CALL_TYPE: Final = "Unknown"
|
||||
INFO_ROUTES_JSON: Final = json.dumps(LiteLLMRoutes.info_routes.value)
|
||||
|
||||
|
||||
class CacheActivityGroup(BaseModel):
|
||||
|
|
@ -69,6 +72,7 @@ GROUPS_SQL: Final = """
|
|||
OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb)))
|
||||
AND ($4::jsonb = '[]'::jsonb
|
||||
OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb)))
|
||||
AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb))
|
||||
GROUP BY 1
|
||||
ORDER BY (COUNT(*)) DESC
|
||||
"""
|
||||
|
|
@ -89,6 +93,7 @@ ERROR_BREAKDOWN_SQL: Final = """
|
|||
OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb)))
|
||||
AND ($4::jsonb = '[]'::jsonb
|
||||
OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb)))
|
||||
AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb))
|
||||
GROUP BY 1, 2, 3
|
||||
ORDER BY (COUNT(*)) DESC
|
||||
"""
|
||||
|
|
@ -100,6 +105,7 @@ KEY_ALIAS_OPTIONS_SQL: Final = """
|
|||
WHERE
|
||||
sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
|
||||
AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb))
|
||||
ORDER BY 1
|
||||
"""
|
||||
|
||||
|
|
@ -110,6 +116,7 @@ MODEL_OPTIONS_SQL: Final = """
|
|||
sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
|
||||
AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
AND sl."model" != ''
|
||||
AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb))
|
||||
ORDER BY 1
|
||||
"""
|
||||
|
||||
|
|
@ -152,10 +159,12 @@ async def get_cache_activity(
|
|||
key_aliases_json: Final = json.dumps(list(key_aliases))
|
||||
models_json: Final = json.dumps(list(models))
|
||||
group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather(
|
||||
prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json),
|
||||
prisma_client.db.query_raw(ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json),
|
||||
prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date),
|
||||
prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date),
|
||||
prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON),
|
||||
prisma_client.db.query_raw(
|
||||
ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON
|
||||
),
|
||||
prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON),
|
||||
prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON),
|
||||
)
|
||||
groups: Final = _groups_adapter.validate_python(group_rows or [])
|
||||
return CacheActivityResponse(
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert
|
||||
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
coerce_token_limit,
|
||||
get_or_create_metadata_bucket,
|
||||
|
|
@ -879,6 +880,18 @@ def _failure_usage_to_lift(
|
|||
_EMPTY_LIFT: Final = MappingProxyType({})
|
||||
|
||||
|
||||
def _call_type_for_route(route: str | None) -> str | None:
|
||||
"""The route's call type when it maps to a single operation (its async and sync variants);
|
||||
None for routes shared by several operations, since the method is not known here."""
|
||||
if route is None:
|
||||
return None
|
||||
call_types: Final = get_call_types_for_route(route)
|
||||
if not call_types:
|
||||
return None
|
||||
operations: Final = frozenset(call_type.value.removeprefix("a") for call_type in call_types)
|
||||
return call_types[0].value if len(operations) == 1 else None
|
||||
|
||||
|
||||
def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""Failure-path callbacks run after ``litellm_logging_obj`` is popped from
|
||||
request_data (it is not serialisable), so the caller merges these fields
|
||||
|
|
@ -2549,10 +2562,6 @@ class ProxyLogging:
|
|||
|
||||
@staticmethod
|
||||
def _stream_requires_guardrail_translation(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
from litellm.litellm_core_utils.api_route_to_call_types import (
|
||||
get_call_types_for_route,
|
||||
)
|
||||
|
||||
route: Final = user_api_key_dict.request_route
|
||||
if not route:
|
||||
return False
|
||||
|
|
@ -3020,6 +3029,7 @@ class ProxyLogging:
|
|||
start_time=datetime.now(),
|
||||
**request_data,
|
||||
)
|
||||
request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller
|
||||
if "metadata" not in request_data:
|
||||
request_data["metadata"] = {}
|
||||
request_data["metadata"].update(user_api_key_logged_metadata)
|
||||
|
|
@ -3044,25 +3054,23 @@ class ProxyLogging:
|
|||
)
|
||||
|
||||
input: list | str | dict = ""
|
||||
normalized_call_type: str | None = None
|
||||
body_shape_call_type: str | None = None
|
||||
if "messages" in request_data and isinstance(request_data["messages"], list):
|
||||
input = request_data["messages"]
|
||||
litellm_logging_obj.model_call_details["messages"] = input
|
||||
if litellm_logging_obj.call_type != CallTypes.pass_through.value:
|
||||
normalized_call_type = CallTypes.acompletion.value
|
||||
body_shape_call_type = CallTypes.acompletion.value
|
||||
elif "prompt" in request_data and isinstance(request_data["prompt"], str):
|
||||
input = request_data["prompt"]
|
||||
litellm_logging_obj.model_call_details["prompt"] = input
|
||||
if litellm_logging_obj.call_type != CallTypes.pass_through.value:
|
||||
normalized_call_type = CallTypes.atext_completion.value
|
||||
body_shape_call_type = CallTypes.atext_completion.value
|
||||
elif "input" in request_data and isinstance(request_data["input"], list):
|
||||
input = request_data["input"]
|
||||
litellm_logging_obj.model_call_details["input"] = input
|
||||
if litellm_logging_obj.call_type != CallTypes.pass_through.value:
|
||||
normalized_call_type = CallTypes.aembedding.value
|
||||
if normalized_call_type is not None:
|
||||
litellm_logging_obj.call_type = normalized_call_type
|
||||
litellm_logging_obj.model_call_details["call_type"] = normalized_call_type
|
||||
body_shape_call_type = CallTypes.aembedding.value
|
||||
resolved_call_type: Final = _call_type_for_route(route) or body_shape_call_type
|
||||
if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value:
|
||||
litellm_logging_obj.call_type = resolved_call_type
|
||||
litellm_logging_obj.model_call_details["call_type"] = resolved_call_type
|
||||
# Pass-through endpoints are logged via the callback loop's
|
||||
# async_post_call_failure_hook — skip pre_call and failure handlers.
|
||||
if litellm_logging_obj.call_type == CallTypes.pass_through.value:
|
||||
|
|
|
|||
102
tests/proxy_behavior/spend/test_cache_activity.py
Normal file
102
tests/proxy_behavior/spend/test_cache_activity.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
"""
|
||||
Behavior tests for the cache analytics queries against a real Postgres. The info-route
|
||||
exclusion and the Unknown grouping live in SQL, so these tests are the ones that exercise
|
||||
them; the endpoint wiring is unit-tested in
|
||||
tests/test_litellm/proxy/analytics_endpoints/test_analytics_endpoints.py.
|
||||
"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.analytics_endpoints.cache_activity import (
|
||||
ERROR_BREAKDOWN_SQL,
|
||||
GROUPS_SQL,
|
||||
INFO_ROUTES_JSON,
|
||||
KEY_ALIAS_OPTIONS_SQL,
|
||||
MODEL_OPTIONS_SQL,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
DAY: Final = datetime(2001, 3, 7)
|
||||
AT_NOON: Final = DAY.replace(hour=12)
|
||||
RUN: Final = uuid.uuid4()
|
||||
INFERENCE_KEY: Final = f"ca-inference-{RUN}"
|
||||
INFO_ONLY_KEY: Final = f"ca-info-only-{RUN}"
|
||||
INFERENCE_ALIAS: Final = f"alias-inference-{RUN}"
|
||||
INFO_ONLY_ALIAS: Final = f"alias-info-only-{RUN}"
|
||||
INFERENCE_MODEL: Final = f"gpt-5.4-mini-{RUN}"
|
||||
INFO_ONLY_MODEL: Final = f"ghost-model-{RUN}"
|
||||
NO_FILTER: Final = "[]"
|
||||
|
||||
|
||||
async def _spend_log(db, api_key: str, call_type: str, status: str, model: str = "", error_code: str = "") -> None:
|
||||
metadata: Final = {"error_information": {"error_code": error_code, "error_class": "ProxyException"}}
|
||||
await db.execute_raw(
|
||||
'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "api_key", "startTime", "endTime", "model", '
|
||||
'"status", "metadata") VALUES ($1, $2, $3, $4::timestamp, $4::timestamp, $5, $6, $7::jsonb)',
|
||||
str(uuid.uuid4()),
|
||||
call_type,
|
||||
api_key,
|
||||
AT_NOON,
|
||||
model,
|
||||
status,
|
||||
json.dumps(metadata if status == "failure" else {}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
async def seeded(db):
|
||||
for token, alias in ((INFERENCE_KEY, INFERENCE_ALIAS), (INFO_ONLY_KEY, INFO_ONLY_ALIAS)):
|
||||
await db.execute_raw(
|
||||
'INSERT INTO "LiteLLM_VerificationToken" ("token", "key_alias") VALUES ($1, $2)', token, alias
|
||||
)
|
||||
await _spend_log(db, INFERENCE_KEY, "acompletion", "success", model=INFERENCE_MODEL)
|
||||
await _spend_log(db, INFERENCE_KEY, "acompletion", "failure", model=INFERENCE_MODEL, error_code="429")
|
||||
await _spend_log(db, INFERENCE_KEY, "", "failure", error_code="401")
|
||||
await _spend_log(db, INFERENCE_KEY, "/model/info", "failure", error_code="401")
|
||||
await _spend_log(db, INFO_ONLY_KEY, "/v1/models", "failure", model=INFO_ONLY_MODEL, error_code="401")
|
||||
await _spend_log(db, INFO_ONLY_KEY, "/key/info", "success")
|
||||
yield
|
||||
keys: Final = [INFERENCE_KEY, INFO_ONLY_KEY]
|
||||
await db.execute_raw('DELETE FROM "LiteLLM_SpendLogs" WHERE "api_key" = ANY($1::text[])', keys)
|
||||
await db.execute_raw('DELETE FROM "LiteLLM_VerificationToken" WHERE "token" = ANY($1::text[])', keys)
|
||||
|
||||
|
||||
async def _groups(db, key_aliases: list[str]) -> dict[str, dict]:
|
||||
rows: Final = await db.query_raw(GROUPS_SQL, DAY, DAY, json.dumps(key_aliases), NO_FILTER, INFO_ROUTES_JSON)
|
||||
return {row["call_type"]: row for row in rows}
|
||||
|
||||
|
||||
async def test_groups_drop_info_routes_and_keep_unknown_for_rows_without_an_endpoint(db):
|
||||
groups: Final = await _groups(db, [INFERENCE_ALIAS])
|
||||
assert set(groups) == {"acompletion", "Unknown"}
|
||||
assert (groups["acompletion"]["api_requests"], groups["acompletion"]["failed_requests"]) == (1, 1)
|
||||
assert (groups["Unknown"]["api_requests"], groups["Unknown"]["failed_requests"]) == (0, 1)
|
||||
|
||||
|
||||
async def test_key_with_only_info_route_traffic_has_no_groups(db):
|
||||
assert await _groups(db, [INFO_ONLY_ALIAS]) == {}
|
||||
|
||||
|
||||
async def test_error_breakdown_drops_info_routes(db):
|
||||
rows: Final = await db.query_raw(
|
||||
ERROR_BREAKDOWN_SQL, DAY, DAY, json.dumps([INFERENCE_ALIAS, INFO_ONLY_ALIAS]), NO_FILTER, INFO_ROUTES_JSON
|
||||
)
|
||||
assert {(row["call_type"], row["error_code"], row["count"]) for row in rows} == {
|
||||
("acompletion", "429", 1),
|
||||
("Unknown", "401", 1),
|
||||
}
|
||||
|
||||
|
||||
async def test_filter_options_only_offer_values_that_return_analytics(db):
|
||||
key_alias_rows: Final = await db.query_raw(KEY_ALIAS_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON)
|
||||
model_rows: Final = await db.query_raw(MODEL_OPTIONS_SQL, DAY, DAY, INFO_ROUTES_JSON)
|
||||
key_aliases: Final = {row["key_alias"] for row in key_alias_rows}
|
||||
models: Final = {row["model"] for row in model_rows}
|
||||
assert INFERENCE_ALIAS in key_aliases and INFO_ONLY_ALIAS not in key_aliases
|
||||
assert INFERENCE_MODEL in models and INFO_ONLY_MODEL not in models
|
||||
|
|
@ -12,10 +12,13 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy._types import LiteLLMRoutes
|
||||
from litellm.proxy.analytics_endpoints.analytics_endpoints import get_global_activity
|
||||
from litellm.proxy.analytics_endpoints.cache_activity import (
|
||||
ERROR_BREAKDOWN_SQL,
|
||||
GROUPS_SQL,
|
||||
KEY_ALIAS_OPTIONS_SQL,
|
||||
MODEL_OPTIONS_SQL,
|
||||
CacheActivityGroup,
|
||||
compute_totals,
|
||||
)
|
||||
|
|
@ -112,6 +115,21 @@ async def test_filters_are_passed_to_sql_as_json_arrays(mock_prisma: MagicMock):
|
|||
assert call.args[4] == json.dumps(["gpt-5.1", "claude-opus-4-8"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_query_excludes_the_same_info_routes(mock_prisma: MagicMock):
|
||||
"""Regression for LIT-5884: failed info-route calls are spend-logged but are not inference traffic, so
|
||||
the groups, error breakdown and both filter-option queries all receive the same exclusion list. What
|
||||
the SQL does with it is covered against Postgres in tests/proxy_behavior/spend/test_cache_activity.py."""
|
||||
await get_global_activity(start_date="2026-07-01", end_date="2026-07-27", key_aliases=[], models=[])
|
||||
|
||||
exclusions_by_query = {call.args[0]: json.loads(call.args[-1]) for call in mock_prisma.db.query_raw.call_args_list}
|
||||
assert set(exclusions_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL}
|
||||
for excluded_call_types in exclusions_by_query.values():
|
||||
assert excluded_call_types == LiteLLMRoutes.info_routes.value
|
||||
assert {"/model/info", "/v1/models", "/key/info"} <= set(excluded_call_types)
|
||||
assert "" not in excluded_call_types
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_malformed_dates_with_400(mock_prisma: MagicMock):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
|
|
|
|||
|
|
@ -33,9 +33,7 @@ def test_is_proxy_only_llm_api_truth_table(proxy_logging):
|
|||
snapshot. Covers no-route, non-LLM route, HTTPException on LLM route,
|
||||
and auth-error short-circuit."""
|
||||
snapshot = {
|
||||
"no_route": proxy_logging._is_proxy_only_llm_api_error(
|
||||
original_exception=Exception(), route=None
|
||||
),
|
||||
"no_route": proxy_logging._is_proxy_only_llm_api_error(original_exception=Exception(), route=None),
|
||||
"non_llm_route": proxy_logging._is_proxy_only_llm_api_error(
|
||||
original_exception=HTTPException(status_code=429, detail="rate"),
|
||||
route="/random/path",
|
||||
|
|
@ -158,9 +156,7 @@ async def test_post_call_failure_hook_non_http_exception_in_callback_swallowed(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_logging_proxy_only_path_uses_existing_logging_obj(
|
||||
proxy_logging, make_user_api_key_auth
|
||||
):
|
||||
async def test_handle_logging_proxy_only_path_uses_existing_logging_obj(proxy_logging, make_user_api_key_auth):
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.call_type = "acompletion"
|
||||
logging_obj.model_call_details = {}
|
||||
|
|
@ -183,10 +179,7 @@ async def test_handle_logging_proxy_only_path_uses_existing_logging_obj(
|
|||
snapshot = {
|
||||
"input_logged": "messages" in logging_obj.model_call_details,
|
||||
"call_type_normalized": logging_obj.call_type,
|
||||
"marker_present": logging_obj.model_call_details.get(
|
||||
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
)
|
||||
is True,
|
||||
"marker_present": logging_obj.model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL) is True,
|
||||
"async_failure_called": logging_obj.async_failure_handler.called,
|
||||
}
|
||||
assert snapshot == {
|
||||
|
|
@ -198,9 +191,7 @@ async def test_handle_logging_proxy_only_path_uses_existing_logging_obj(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_logging_proxy_only_path_skips_for_pass_through(
|
||||
proxy_logging, make_user_api_key_auth
|
||||
):
|
||||
async def test_handle_logging_proxy_only_path_skips_for_pass_through(proxy_logging, make_user_api_key_auth):
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
logging_obj = MagicMock()
|
||||
|
|
@ -248,9 +239,7 @@ async def test_handle_logging_proxy_only_path_no_logging_obj_creates_one(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_logging_proxy_only_path_propagates_async_failure_raises(
|
||||
proxy_logging, make_user_api_key_auth
|
||||
):
|
||||
async def test_handle_logging_proxy_only_path_propagates_async_failure_raises(proxy_logging, make_user_api_key_auth):
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.call_type = "acompletion"
|
||||
logging_obj.model_call_details = {}
|
||||
|
|
@ -267,3 +256,65 @@ async def test_handle_logging_proxy_only_path_propagates_async_failure_raises(
|
|||
route="/chat/completions",
|
||||
original_exception=Exception("x"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"route, request_data, expected_call_type",
|
||||
[
|
||||
("/v1/chat/completions", {}, "acompletion"),
|
||||
("/chat/completions", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}, "acompletion"),
|
||||
("/v1/messages", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}, "anthropic_messages"),
|
||||
("/v1/responses", {"model": "m", "input": "hi"}, "aresponses"),
|
||||
("/v1/embeddings", {"model": "m", "input": ["hi"]}, "aembedding"),
|
||||
("/model/info", {}, "/model/info"),
|
||||
],
|
||||
)
|
||||
async def test_post_call_failure_hook_lifts_route_call_type_for_gate_rejections(
|
||||
proxy_logging, make_user_api_key_auth, route, request_data, expected_call_type
|
||||
):
|
||||
"""Regression for LIT-5884: the matched route, not the body shape, sets the
|
||||
spend-log call_type for requests rejected before dispatch."""
|
||||
proxy_logging.alert_types = []
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=Exception("Authentication Error, No api key passed in."),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route=route),
|
||||
error_type=ProxyErrorTypes.auth_error,
|
||||
route=route,
|
||||
)
|
||||
assert request_data["call_type"] == expected_call_type
|
||||
assert "start_time" in request_data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_falls_back_to_body_shape_without_a_route(proxy_logging, make_user_api_key_auth):
|
||||
proxy_logging.alert_types = []
|
||||
request_data = {"model": "m", "messages": [{"role": "user", "content": "hi"}]}
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=Exception("Authentication Error, No api key passed in."),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
error_type=ProxyErrorTypes.auth_error,
|
||||
)
|
||||
assert request_data["call_type"] == "acompletion"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", ["/v1/files", "/files/file-abc", "/v1/containers"])
|
||||
async def test_post_call_failure_hook_keeps_the_route_for_multi_operation_routes(
|
||||
proxy_logging, make_user_api_key_auth, route
|
||||
):
|
||||
"""Routes shared by several operations (POST create vs GET list) cannot be attributed without the
|
||||
method, so a rejected request there is filed under its route, not under whichever operation the
|
||||
mapping lists first."""
|
||||
proxy_logging.alert_types = []
|
||||
request_data: dict = {}
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=Exception("Authentication Error, No api key passed in."),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route=route),
|
||||
error_type=ProxyErrorTypes.auth_error,
|
||||
route=route,
|
||||
)
|
||||
assert request_data["call_type"] == route
|
||||
|
|
|
|||
|
|
@ -236,6 +236,35 @@ describe("CacheDashboard cache analytics charts", () => {
|
|||
expect(screen.queryByText(/Failed requests by error code/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("explains the Unknown bucket only when a group has no recorded endpoint", async () => {
|
||||
const { rerender } = renderDashboard();
|
||||
await screen.findByText(REQUESTS_CHART_TITLE);
|
||||
expect(screen.queryByText(/recorded no endpoint/)).not.toBeInTheDocument();
|
||||
|
||||
useCacheActivity.mockReturnValue({
|
||||
data: {
|
||||
...cacheActivity,
|
||||
groups: [
|
||||
...cacheActivity.groups,
|
||||
{
|
||||
call_type: "Unknown",
|
||||
api_requests: 0,
|
||||
cache_hits: 0,
|
||||
failed_requests: 121000,
|
||||
cached_completion_tokens: 0,
|
||||
generated_completion_tokens: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
refetch: vi.fn(),
|
||||
});
|
||||
rerender(<CacheDashboard accessToken="sk-test" token="tok" userRole="Admin" userID="u1" premiumUser={false} />);
|
||||
|
||||
expect(
|
||||
within(cardTitled(REQUESTS_CHART_TITLE)).getByText(/Unknown groups spend logs that recorded no endpoint/),
|
||||
).toHaveTextContent("not necessarily LLM API requests");
|
||||
});
|
||||
|
||||
it("formats y-axis ticks with compact notation", async () => {
|
||||
renderDashboard();
|
||||
const { requestsCard, tokensCard } = await findChartCards();
|
||||
|
|
|
|||
|
|
@ -35,6 +35,11 @@ const REQUEST_SERIES = {
|
|||
failed: "Failed requests",
|
||||
} as const;
|
||||
|
||||
const UNKNOWN_CALL_TYPE = "Unknown";
|
||||
|
||||
const UNKNOWN_CALL_TYPE_NOTE =
|
||||
"Unknown groups spend logs that recorded no endpoint. Older proxy versions wrote those for requests rejected before routing, so they are not necessarily LLM API requests.";
|
||||
|
||||
const toChartDatum = (group: CacheActivityGroup) => ({
|
||||
name: group.call_type,
|
||||
[REQUEST_SERIES.apiRequests]: group.api_requests,
|
||||
|
|
@ -103,6 +108,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
|
|||
const uniqueApiKeys = activity?.filter_options.key_aliases ?? [];
|
||||
const uniqueModels = activity?.filter_options.models ?? [];
|
||||
const chartData = (activity?.groups ?? []).map(toChartDatum);
|
||||
const hasUnknownGroup = (activity?.groups ?? []).some((group) => group.call_type === UNKNOWN_CALL_TYPE);
|
||||
const activeDrilldownCallType = resolveDrilldownCallType(errorDrilldownCallType, activity?.groups ?? []);
|
||||
|
||||
const handleRefreshClick = () => {
|
||||
|
|
@ -288,6 +294,7 @@ const CacheDashboard: React.FC<CachePageProps> = ({ accessToken, token, userRole
|
|||
<p className="text-sm text-muted-foreground">
|
||||
Click a red failed-requests segment to see which error codes caused those failures.
|
||||
</p>
|
||||
{hasUnknownGroup && <p className="mt-1 text-sm text-muted-foreground">{UNKNOWN_CALL_TYPE_NOTE}</p>}
|
||||
<BarChart
|
||||
data={chartData}
|
||||
stack={true}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue