fix(proxy): keep router cooldown 429s at ERROR and drop tracebacks from quiet auth warnings

A 429 because every deployment in a group is cooling down or at its own limit
is a capacity fault the operator has to act on, so it keeps logging at ERROR.
The auth handler no longer attaches a traceback to a quiet WARNING, and the
call-id tests now capture the WARNING a 401 logs.
This commit is contained in:
mateo-berri 2026-11-02 21:16:18 -08:00
parent a1ab66a56c
commit 2cf410532d
8 changed files with 34 additions and 10 deletions

View file

@ -668,10 +668,10 @@ def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
class LevelRoutingStreamHandler(logging.StreamHandler):
"""Writes records below WARNING and invalid-key warnings to stdout, others to stderr.
"""Writes records below WARNING and client-caused failure warnings to stdout, others to stderr.
Collectors that derive severity from the stream report every stderr line as an error.
Invalid-key warnings route to stdout so LITELLM_LOG=ERROR can suppress them.
Client-caused failure warnings route to stdout so LITELLM_LOG=ERROR can suppress them.
"""
def emit(self, record: logging.LogRecord) -> None:
@ -937,7 +937,7 @@ else:
handler.setFormatter(formatter)
verbose_proxy_logger = logging.getLogger("LiteLLM Proxy")
# Malformed virtual key rejections log through this child; LevelRoutingStreamHandler
# Client-caused request failures log through this child; LevelRoutingStreamHandler
# writes its WARNING records to stdout. It has no handler or level of its own.
verbose_proxy_stdout_logger: Final = verbose_proxy_logger.getChild("stdout")
verbose_router_logger = logging.getLogger("LiteLLM Router")

View file

@ -273,12 +273,16 @@ def is_client_caused_failure(exception: BaseException | None) -> bool:
instead of an ERROR an operator gets paged on: the proxy's own 4xx rejections and a provider
saying the request itself is invalid (400, 413, 422). A guardrail that could not reach a
verdict, a 4xx from a backend the proxy called itself (``httpx.HTTPStatusError``, e.g. a
moderation API rejecting the proxy's own key), and provider auth, permission, not-found and
rate-limit errors stay faults.
moderation API rejecting the proxy's own key), a router 429 because every deployment is
cooling down or at its own limit, and provider auth, permission, not-found and rate-limit
errors stay faults.
"""
from litellm.exceptions import GuardrailRaisedException
from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic
if exception is None or isinstance(exception, httpx.HTTPStatusError):
if exception is None or isinstance(
exception, (httpx.HTTPStatusError, RouterRateLimitError, RouterRateLimitErrorBasic)
):
return False
if isinstance(exception, GuardrailRaisedException) and not exception.blocked_content:
return False

View file

@ -218,7 +218,11 @@ class UserAPIKeyAuthExceptionHandler:
_auth_failure_log_message(e),
requester_ip,
_identity_log_suffix(resolved_identity),
exc_info=True if litellm.log_client_error_tracebacks or not is_expected_client_error(e) else None,
exc_info=(
True
if not is_quiet_log and (litellm.log_client_error_tracebacks or not is_expected_client_error(e))
else None
),
extra=log_extra,
)

View file

@ -1189,6 +1189,12 @@ def _marked_malformed_key_error() -> HTTPException:
"WARNING",
id="key_over_rate_limit_warning_no_traceback",
),
pytest.param(
litellm.BadRequestError(message="Invalid request body", model="gpt-5.4-mini", llm_provider="openai"),
False,
"WARNING",
id="provider_400_warning_no_traceback",
),
pytest.param(
ProxyException(message="DB unavailable", type=ProxyErrorTypes.auth_error, param=None, code=503),
True,

View file

@ -350,7 +350,7 @@ async def test_failure_log_carries_the_callers_litellm_call_id(
receive,
)
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
await endpoints.image_generation(
request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth()
)

View file

@ -8966,7 +8966,7 @@ async def test_chat_completion_pass_through_endpoint_failure_carries_the_callers
return_value=json.dumps({"model": "unknown-model", "messages": [{"role": "user", "content": "hi"}]}).encode()
)
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
await chat_completion_pass_through_endpoint(
fastapi_response=Response(),
request=request,

View file

@ -175,7 +175,7 @@ async def test_failure_log_carries_the_callers_litellm_call_id(
call_id = "rerank-call-7836"
failure = HTTPException(status_code=401, detail={"error": "invalid api key"})
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
raised = await _rerank_failure(
failure,
raised_before_routing=False,

View file

@ -11061,6 +11061,16 @@ def _moderation_backend_rejected_proxy_key() -> Exception:
pytest.param(
ValueError("unexpected internal error"), False, "ERROR", True, id="unexpected_error_with_traceback"
),
pytest.param(
RouterRateLimitError(model="gpt-4", cooldown_time=42.3, enable_pre_call_checks=False, cooldown_list=[]),
False,
"ERROR",
False,
id="every_deployment_cooling_down_error",
),
pytest.param(
RouterRateLimitErrorBasic(model="gpt-4"), False, "ERROR", False, id="no_deployment_available_error"
),
pytest.param(
HTTPException(status_code=400, detail="Invalid model name passed in"),
True,