diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index 8a868b3aff4..50813500813 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -19,6 +19,7 @@ ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = { 403: "permission_error", 404: "not_found_error", 413: "request_too_large", + 422: "invalid_request_error", 429: "rate_limit_error", 500: "api_error", 529: "overloaded_error", @@ -34,16 +35,8 @@ class AnthropicExceptionMapping: @staticmethod def get_error_type(status_code: int) -> AnthropicErrorType: - """Map HTTP status code to Anthropic error type. - - A 4xx the table does not name is the caller's mistake, so it answers - ``invalid_request_error`` rather than ``api_error``, which the Anthropic taxonomy - reserves for server-side failures. - """ - mapped: Final = ANTHROPIC_ERROR_TYPE_MAP.get(status_code) - if mapped is not None: - return mapped - return "invalid_request_error" if 400 <= status_code < 500 else "api_error" + """Map HTTP status code to Anthropic error type.""" + return ANTHROPIC_ERROR_TYPE_MAP.get(status_code, "api_error") @staticmethod def create_error_response( diff --git a/tests/integration/authorization/_hidden_alias_budget.py b/tests/integration/authorization/_hidden_alias_budget.py index eb099e56026..fb3cd51131c 100644 --- a/tests/integration/authorization/_hidden_alias_budget.py +++ b/tests/integration/authorization/_hidden_alias_budget.py @@ -150,6 +150,13 @@ def error_type(response: httpx.Response) -> str: return str(object_value(JSON_OBJECT.validate_json(response.content)["error"])["type"]) +def budget_refusal_error_type(route: str) -> str: + """The ``error.type`` a budget refusal carries on a route label from ``ROUTES``: the + Anthropic envelope on Messages (its budget refusal is raised in + ``user_api_key_auth``), the OpenAI ``budget_exceeded`` type on the other routes.""" + return "invalid_request_error" if route == "messages" else "budget_exceeded" + + def settle( send: Callable[[str], httpx.Response], status: int, *, burst: int = GATEWAY_BURST, seconds: float = 60 ) -> None: diff --git a/tests/integration/authorization/test_alias_chain_budget_waiver.py b/tests/integration/authorization/test_alias_chain_budget_waiver.py index 0d64415d445..4dbcc361bc3 100644 --- a/tests/integration/authorization/test_alias_chain_budget_waiver.py +++ b/tests/integration/authorization/test_alias_chain_budget_waiver.py @@ -44,6 +44,7 @@ from integration.authorization._hidden_alias_budget import ( async_anthropic_client, async_openai_client, base_url, + budget_refusal_error_type, error_type, fresh_chat, fresh_message, @@ -191,17 +192,18 @@ def _assert_refused( key: str, model: str, prefix: str, + route: str = "chat", send: Callable[[Gateway, str, str, str], httpx.Response] = fresh_chat, ) -> None: marker: Final = f"{prefix}-" + uuid.uuid4().hex refused: Final = send(candidate, model, key, marker) - assert _refusal(refused) == (BUDGET_EXCEEDED, "budget_exceeded"), refused.text + assert _refusal(refused) == (BUDGET_EXCEEDED, budget_refusal_error_type(route)), refused.text assert upstream_hits(upstream_requests(rig.gateway.upstream_url), marker) == 0 def _assert_refused_on_every_route(rig: ChainRig, candidate: Gateway, key: str, model: str) -> None: for route, send in ROUTES: - _assert_refused(rig, candidate, key, model, f"refused-{route}", send) + _assert_refused(rig, candidate, key, model, f"refused-{route}", route, send) def _assert_provider_failure(response: httpx.Response) -> None: diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index acab042bfc9..5be4379ab2e 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -447,15 +447,14 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be ) assert model_b_response.status_code == 403, model_b_response.text assert _JSON_OBJECT.validate_json(model_b_response.content) == { + "type": "error", "error": { + "type": "permission_error", "message": ( f"The requested model '{model_b}' is not available for this API key, or the model name is " "invalid. Check the models available to you and try again." ), - "type": "team_model_access_denied", - "param": "model", - "code": "403", - } + }, }, model_b_response.text assert upstream.drain() == () diff --git a/tests/integration/authorization/test_cli_sso_login_ui_disabled.py b/tests/integration/authorization/test_cli_sso_login_ui_disabled.py index 315a0b20334..c908b9896e8 100644 --- a/tests/integration/authorization/test_cli_sso_login_ui_disabled.py +++ b/tests/integration/authorization/test_cli_sso_login_ui_disabled.py @@ -695,9 +695,11 @@ def test_cli_session_token_is_denied_once_its_team_budget_is_exhausted( key=key, ) assert refused.status_code == 422, f"{refused.status_code} {refused.text}" - error: Final = object_value(JSON_OBJECT.validate_json(refused.content)["error"]) - assert error["type"] == "budget_exceeded", refused.text - assert error["code"] == "422", refused.text + body: Final = JSON_OBJECT.validate_json(refused.content) + assert body["type"] == "error", refused.text + error: Final = object_value(body["error"]) + assert error["type"] == "invalid_request_error", refused.text + assert "param" not in error and "code" not in error, refused.text message: Final = string_value(error["message"]) assert "Budget has been exceeded!" in message, refused.text assert f"Team={team}" in message, refused.text diff --git a/tests/integration/authorization/test_hidden_alias_budget_bypass.py b/tests/integration/authorization/test_hidden_alias_budget_bypass.py index 493db4c3e62..f624aa92048 100644 --- a/tests/integration/authorization/test_hidden_alias_budget_bypass.py +++ b/tests/integration/authorization/test_hidden_alias_budget_bypass.py @@ -38,6 +38,7 @@ from integration.authorization._hidden_alias_budget import ( async_anthropic_client, async_openai_client, base_url, + budget_refusal_error_type, chat_statuses, clear_provider_script, error_type, @@ -599,7 +600,7 @@ def _assert_refused_like_target(rig: AliasRig, candidate: Gateway, key: str, sha target_marker: Final = f"target-{route}-" + uuid.uuid4().hex via_shadow: Final = send(candidate, shadow, key, shadow_marker) by_name: Final = send(candidate, target, key, target_marker) - assert _refusal(by_name) == (BUDGET_EXCEEDED, "budget_exceeded"), by_name.text + assert _refusal(by_name) == (BUDGET_EXCEEDED, budget_refusal_error_type(route)), by_name.text assert via_shadow.status_code == by_name.status_code, via_shadow.text assert error_type(via_shadow) == error_type(by_name), via_shadow.text observed: Final = upstream_requests(rig.gateway.upstream_url) @@ -621,7 +622,7 @@ def _assert_refused_on_every_route(rig: AliasRig, candidate: Gateway, key: str, marker: Final = f"chain-{route}-" + uuid.uuid4().hex refused: Final = send(candidate, model, key, marker) assert refused.status_code == BUDGET_EXCEEDED, refused.text - assert error_type(refused) == "budget_exceeded", refused.text + assert error_type(refused) == budget_refusal_error_type(route), refused.text assert upstream_hits(upstream_requests(rig.gateway.upstream_url), marker) == 0 @@ -828,9 +829,9 @@ def _deployment_id(rig: AliasRig, name: str) -> str: return identities[0] -def _assert_budget_refusal(status_code: int, body: str) -> None: +def _assert_budget_refusal(status_code: int, body: str, route: str = "chat") -> None: assert status_code == BUDGET_EXCEEDED, body - assert "budget_exceeded" in body, body + assert budget_refusal_error_type(route) in body, body def _assert_no_served_row(key: str, marker: str) -> None: @@ -881,7 +882,7 @@ async def test_exhausted_key_is_refused_on_shadowing_alias_through_streamed_anth extra_headers=spend_marker(marker), ) as stream: await stream.get_final_message() - _assert_budget_refusal(refused.value.status_code, str(refused.value.body)) + _assert_budget_refusal(refused.value.status_code, str(refused.value.body), route="messages") assert upstream_hits(upstream_requests(rig.gateway.upstream_url), marker) == 0 diff --git a/tests/integration/authorization/test_unknown_key_generic_401.py b/tests/integration/authorization/test_unknown_key_generic_401.py index 8e6ff3fa3ab..7826c11bec8 100644 --- a/tests/integration/authorization/test_unknown_key_generic_401.py +++ b/tests/integration/authorization/test_unknown_key_generic_401.py @@ -50,7 +50,23 @@ class _ErrorResponse(BaseModel): error: _ProxyError +class _AnthropicErrorDetail(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + type: str + message: str + + +class _AnthropicErrorResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + type: str + error: _AnthropicErrorDetail + + GENERIC_ERROR: Final = _ProxyError(message=GENERIC_MESSAGE, type="token_not_found_in_db", param="key", code="401") +MESSAGES_PATH: Final = "/v1/messages" +ANTHROPIC_GENERIC_ERROR: Final = _AnthropicErrorDetail(type="authentication_error", message=GENERIC_MESSAGE) @dataclass(frozen=True, slots=True) @@ -119,6 +135,23 @@ def _assert_generic_refusal(response: httpx.Response, *secrets: str) -> None: assert secret not in response.text, response.text +def _assert_messages_refusal(response: httpx.Response, *secrets: str) -> None: + assert response.status_code == 401, response.text + body: Final = _AnthropicErrorResponse.model_validate_json(response.content) + assert body.type == "error", response.text + assert body.error == ANTHROPIC_GENERIC_ERROR, response.text + for secret in secrets: + assert secret not in response.text, response.text + + +def _assert_refusal(path: str, response: httpx.Response, *secrets: str) -> None: + """The Messages routes answer the Anthropic envelope, every other route the OpenAI one.""" + if path == MESSAGES_PATH: + _assert_messages_refusal(response, *secrets) + else: + _assert_generic_refusal(response, *secrets) + + def _side(gateway: Gateway, peer: Gateway, side: Side) -> Gateway: return gateway if side == "gateway" else peer @@ -129,7 +162,10 @@ def test_an_unknown_key_gets_the_generic_401_without_reaching_the_provider( gateway: Gateway, peer: Gateway, provider: SharedProvider, route: str, side: Side ) -> None: key: Final = _unknown_key() - _assert_generic_refusal(_send(_side(gateway, peer, side), ROUTES[route], key), key, key[-4:], _digest(key)) + request_route: Final = ROUTES[route] + _assert_refusal( + request_route.path, _send(_side(gateway, peer, side), request_route, key), key, key[-4:], _digest(key) + ) assert provider.received() == () @@ -183,8 +219,9 @@ def test_two_different_unknown_keys_in_both_headers_get_the_generic_401_naming_n bearer: Final = _unknown_key() api_key: Final = _unknown_key() headers: Final = {**_headers("authorization", bearer), **_headers("x-api-key", api_key)} - for path, body in (("/v1/chat/completions", CHAT), ("/v1/messages", MESSAGES)): - _assert_generic_refusal( + for path, body in (("/v1/chat/completions", CHAT), (MESSAGES_PATH, MESSAGES)): + _assert_refusal( + path, gateway.client.post(path, json=dict(body), headers=headers), bearer, api_key, @@ -359,16 +396,15 @@ async def _send_async(client: httpx.AsyncClient, route: _Route, key: str) -> htt ) -async def _burst(owned: OwnedProxy, keys: tuple[str, ...]) -> tuple[httpx.Response, ...]: +async def _burst(owned: OwnedProxy, keys: tuple[str, ...]) -> tuple[tuple[_Route, httpx.Response], ...]: + """Each key's response paired with the route it was sent on, so the caller asserts the + envelope the route actually answered with.""" + routes: Final = tuple(ROUTES[BURST_ROUTES[index % len(BURST_ROUTES)]] for index in range(len(keys))) async with httpx.AsyncClient(base_url=str(owned.gateway.client.base_url), timeout=60, trust_env=False) as client: - return tuple( - await asyncio.gather( - *( - _send_async(client, ROUTES[BURST_ROUTES[index % len(BURST_ROUTES)]], key) - for index, key in enumerate(keys) - ) - ) + responses: Final = await asyncio.gather( + *(_send_async(client, route, key) for route, key in zip(routes, keys, strict=True)) ) + return tuple(zip(routes, responses, strict=True)) def _log_after(owned: OwnedProxy, offset: int) -> str: @@ -397,7 +433,7 @@ async def test_a_burst_across_a_worker_kill_answers_every_unknown_key_with_the_g lambda text: all(_console_line(key) in text for key in keys), 30, ) - for response, key in zip((*during, *after), keys, strict=True): - _assert_generic_refusal(response, key, _digest(key)) + for (route, response), key in zip((*during, *after), keys, strict=True): + _assert_refusal(route.path, response, key, _digest(key)) assert [window.count(_console_line(key)) for key in keys] == [1] * len(keys) assert [key for key in keys if key in window] == [] diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py index 8a79d5c6a7a..91748187876 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py @@ -263,15 +263,14 @@ def test_enabled_gateway_count_tokens_alias_matches_native_count_tokens_route(en ) assert restricted_response.status_code == 403, restricted_response.text assert _JSON_OBJECT.validate_json(restricted_response.content) == { + "type": "error", "error": { + "type": "permission_error", "message": ( f"The requested model '{restricted_model}' is not available for this API key, or the model name " "is invalid. Check the models available to you and try again." ), - "type": "key_model_access_denied", - "param": "model", - "code": "403", - } + }, }, restricted_response.text assert wire.drain() == () @@ -280,15 +279,14 @@ def test_enabled_gateway_count_tokens_alias_matches_native_count_tokens_route(en def test_enabled_gateway_refuses_model_outside_virtual_key(enabled_gateway: Gateway) -> None: result: Final = _restricted_model_responses(enabled_gateway) expected_body: Final = { + "type": "error", "error": { + "type": "permission_error", "message": ( f"The requested model '{result[4]}' is not available for this API key, or the model name is invalid. " "Check the models available to you and try again." ), - "type": "key_model_access_denied", - "param": "model", - "code": "403", - } + }, } assert result[0] == result[2] == 403, (result[1], result[3]) assert _JSON_OBJECT.validate_json(result[1]) == expected_body, result[1] diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index 29b18f1f261..0a96709f874 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -258,9 +258,20 @@ def _observed(upstream: httpx.Client) -> list[JsonValue]: return requests -def _denial_error(denied: httpx.Response) -> dict[str, JsonValue]: +def _denial_error(denied: httpx.Response, route: str) -> dict[str, JsonValue]: assert denied.status_code == 422, denied.text - return object_value(object_value(denied.json())["error"]) + if route == "/v1/messages": + body: Final = object_value(denied.json()) + assert body["type"] == "error", denied.text + anthropic_error: Final = object_value(body["error"]) + assert anthropic_error["type"] == "invalid_request_error", denied.text + assert "param" not in anthropic_error and "code" not in anthropic_error, denied.text + return anthropic_error + openai_error: Final = object_value(object_value(denied.json())["error"]) + assert openai_error["type"] == "budget_exceeded", denied.text + assert openai_error["param"] is None + assert openai_error["code"] == "422" + return openai_error def _prove_end_user_budget_lockout( @@ -304,8 +315,7 @@ def _prove_end_user_budget_lockout( _observed(upstream) denied: Final = _send(gateway, spelling, deployment, key, end_user_id, f"denied {end_user_id}") assert _observed(upstream) == [], denied.text - error: Final = _denial_error(denied) - assert error["type"] == "budget_exceeded", denied.text + error: Final = _denial_error(denied, spelling.route) matched: Final = re.fullmatch( rf"ExceededBudget: End User={re.escape(end_user_id)} over budget\. " rf"Spend=([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?), Budget=0\.05", diff --git a/tests/integration/spend/test_end_user_spend_without_proxy_user.py b/tests/integration/spend/test_end_user_spend_without_proxy_user.py index 16210438e82..9d5b82f5c10 100644 --- a/tests/integration/spend/test_end_user_spend_without_proxy_user.py +++ b/tests/integration/spend/test_end_user_spend_without_proxy_user.py @@ -118,6 +118,19 @@ def _assert_budget_refusal(response: httpx.Response, end_user: str) -> None: ] +def _assert_messages_budget_refusal(response: httpx.Response, end_user: str) -> None: + """A budget refusal on /v1/messages keeps its 422 but answers the Anthropic envelope.""" + assert response.status_code == 422, response.text + body: Final = _json_object(response.content) + assert body["type"] == "error", response.text + error: Final = object_value(body["error"]) + assert error["type"] == "invalid_request_error" + assert "param" not in error and "code" not in error, response.text + assert str(error["message"]).startswith(f"ExceededBudget: End User={end_user} over budget. Spend="), error[ + "message" + ] + + def _json_object(raw: bytes) -> dict[str, JsonValue]: return _JSON_OBJECT.validate_json(raw) @@ -341,7 +354,7 @@ def test_anthropic_messages_metadata_user_id_is_charged_and_budget_enforced(gate json={**body, "messages": [{"role": "user", "content": f"again {uuid.uuid4().hex}"}]}, headers={"x-api-key": key, "anthropic-version": "2023-06-01"}, ) - _assert_budget_refusal(denied, end_user) + _assert_messages_budget_refusal(denied, end_user) assert _observed(upstream) == [] diff --git a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py index 6d13b91aab5..5c40bfafa08 100644 --- a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py +++ b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py @@ -65,12 +65,14 @@ class TestCreateErrorResponse: @pytest.mark.parametrize( ("status_code", "expected_error_type"), - [(418, "invalid_request_error"), (422, "invalid_request_error"), (502, "api_error"), (599, "api_error")], + [(422, "invalid_request_error"), (418, "api_error"), (502, "api_error"), (599, "api_error")], ) - def test_an_unmapped_status_is_the_callers_fault_when_4xx(self, status_code: int, expected_error_type: str): - """An unmapped 4xx is the client's mistake, so it must not answer api_error, which - the Anthropic taxonomy reserves for server-side failures; 5xx keeps api_error.""" - response = AnthropicExceptionMapping.create_error_response(status_code, "mapped only by range") + def test_a_budget_status_is_a_request_error_and_a_status_the_table_does_not_name_is_api_error( + self, status_code: int, expected_error_type: str + ): + """The proxy's budget status is a caller-fault request error, so the table names 422. + A status the table does not name, such as a 418 or a 5xx, keeps api_error.""" + response = AnthropicExceptionMapping.create_error_response(status_code, "scripted status") assert response["error"]["type"] == expected_error_type