mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): keep router cooldown 429s at ERROR and drop tracebacks from quiet auth warnings
A 429 because every deployment in a group is cooling down or at its own limit is a capacity fault the operator has to act on, so it keeps logging at ERROR. The auth handler no longer attaches a traceback to a quiet WARNING, and the call-id tests now capture the WARNING a 401 logs.
This commit is contained in:
parent
a1ab66a56c
commit
2cf410532d
8 changed files with 34 additions and 10 deletions
|
|
@ -668,10 +668,10 @@ def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
|
|||
|
||||
|
||||
class LevelRoutingStreamHandler(logging.StreamHandler):
|
||||
"""Writes records below WARNING and invalid-key warnings to stdout, others to stderr.
|
||||
"""Writes records below WARNING and client-caused failure warnings to stdout, others to stderr.
|
||||
|
||||
Collectors that derive severity from the stream report every stderr line as an error.
|
||||
Invalid-key warnings route to stdout so LITELLM_LOG=ERROR can suppress them.
|
||||
Client-caused failure warnings route to stdout so LITELLM_LOG=ERROR can suppress them.
|
||||
"""
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
|
|
@ -937,7 +937,7 @@ else:
|
|||
handler.setFormatter(formatter)
|
||||
|
||||
verbose_proxy_logger = logging.getLogger("LiteLLM Proxy")
|
||||
# Malformed virtual key rejections log through this child; LevelRoutingStreamHandler
|
||||
# Client-caused request failures log through this child; LevelRoutingStreamHandler
|
||||
# writes its WARNING records to stdout. It has no handler or level of its own.
|
||||
verbose_proxy_stdout_logger: Final = verbose_proxy_logger.getChild("stdout")
|
||||
verbose_router_logger = logging.getLogger("LiteLLM Router")
|
||||
|
|
|
|||
|
|
@ -273,12 +273,16 @@ def is_client_caused_failure(exception: BaseException | None) -> bool:
|
|||
instead of an ERROR an operator gets paged on: the proxy's own 4xx rejections and a provider
|
||||
saying the request itself is invalid (400, 413, 422). A guardrail that could not reach a
|
||||
verdict, a 4xx from a backend the proxy called itself (``httpx.HTTPStatusError``, e.g. a
|
||||
moderation API rejecting the proxy's own key), and provider auth, permission, not-found and
|
||||
rate-limit errors stay faults.
|
||||
moderation API rejecting the proxy's own key), a router 429 because every deployment is
|
||||
cooling down or at its own limit, and provider auth, permission, not-found and rate-limit
|
||||
errors stay faults.
|
||||
"""
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic
|
||||
|
||||
if exception is None or isinstance(exception, httpx.HTTPStatusError):
|
||||
if exception is None or isinstance(
|
||||
exception, (httpx.HTTPStatusError, RouterRateLimitError, RouterRateLimitErrorBasic)
|
||||
):
|
||||
return False
|
||||
if isinstance(exception, GuardrailRaisedException) and not exception.blocked_content:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -218,7 +218,11 @@ class UserAPIKeyAuthExceptionHandler:
|
|||
_auth_failure_log_message(e),
|
||||
requester_ip,
|
||||
_identity_log_suffix(resolved_identity),
|
||||
exc_info=True if litellm.log_client_error_tracebacks or not is_expected_client_error(e) else None,
|
||||
exc_info=(
|
||||
True
|
||||
if not is_quiet_log and (litellm.log_client_error_tracebacks or not is_expected_client_error(e))
|
||||
else None
|
||||
),
|
||||
extra=log_extra,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1189,6 +1189,12 @@ def _marked_malformed_key_error() -> HTTPException:
|
|||
"WARNING",
|
||||
id="key_over_rate_limit_warning_no_traceback",
|
||||
),
|
||||
pytest.param(
|
||||
litellm.BadRequestError(message="Invalid request body", model="gpt-5.4-mini", llm_provider="openai"),
|
||||
False,
|
||||
"WARNING",
|
||||
id="provider_400_warning_no_traceback",
|
||||
),
|
||||
pytest.param(
|
||||
ProxyException(message="DB unavailable", type=ProxyErrorTypes.auth_error, param=None, code=503),
|
||||
True,
|
||||
|
|
|
|||
|
|
@ -350,7 +350,7 @@ async def test_failure_log_carries_the_callers_litellm_call_id(
|
|||
receive,
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
|
||||
await endpoints.image_generation(
|
||||
request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth()
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8966,7 +8966,7 @@ async def test_chat_completion_pass_through_endpoint_failure_carries_the_callers
|
|||
return_value=json.dumps({"model": "unknown-model", "messages": [{"role": "user", "content": "hi"}]}).encode()
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised:
|
||||
await chat_completion_pass_through_endpoint(
|
||||
fastapi_response=Response(),
|
||||
request=request,
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ async def test_failure_log_carries_the_callers_litellm_call_id(
|
|||
call_id = "rerank-call-7836"
|
||||
failure = HTTPException(status_code=401, detail={"error": "invalid api key"})
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
raised = await _rerank_failure(
|
||||
failure,
|
||||
raised_before_routing=False,
|
||||
|
|
|
|||
|
|
@ -11061,6 +11061,16 @@ def _moderation_backend_rejected_proxy_key() -> Exception:
|
|||
pytest.param(
|
||||
ValueError("unexpected internal error"), False, "ERROR", True, id="unexpected_error_with_traceback"
|
||||
),
|
||||
pytest.param(
|
||||
RouterRateLimitError(model="gpt-4", cooldown_time=42.3, enable_pre_call_checks=False, cooldown_list=[]),
|
||||
False,
|
||||
"ERROR",
|
||||
False,
|
||||
id="every_deployment_cooling_down_error",
|
||||
),
|
||||
pytest.param(
|
||||
RouterRateLimitErrorBasic(model="gpt-4"), False, "ERROR", False, id="no_deployment_available_error"
|
||||
),
|
||||
pytest.param(
|
||||
HTTPException(status_code=400, detail="Invalid model name passed in"),
|
||||
True,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue