diff --git a/litellm/constants.py b/litellm/constants.py index 75f12190b3e..6f0e9e7afe2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_non DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) +ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) diff --git a/litellm/router.py b/litellm/router.py index 1edb80da7ce..1a76eb5d59b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -109,6 +109,7 @@ from litellm.router_utils.common_utils import ( filter_team_based_models, filter_web_search_deployments, resolve_model_group_alias, + truncate_fallback_error_detail, ) from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.cooldown_handlers import ( @@ -342,6 +343,12 @@ def _replay_live_router_model_cost() -> None: set_live_deployment_replay(_replay_live_router_model_cost) +# Kwargs that log_retry must not copy into a retry breadcrumb. The breadcrumbs reach spend +# logs and logging callbacks, and these carry either the request payload or router-internal +# walk state rather than anything that identifies the failed attempt. +RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset(("messages", "original_function", "attempted_targets")) + + class Router: model_names: set = set() cache_responses: bool | None = False @@ -6361,17 +6368,16 @@ class Router: return response except Exception as new_exception: parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs) - fallback_failure_exception_str = redact_string(str(new_exception)) + fallback_failure_exception_str = truncate_fallback_error_detail(redact_string(str(new_exception))) cooldown_info: Final = await _async_get_cooldown_deployments_with_debug_info( litellm_router_instance=self, parent_otel_span=parent_otel_span, ) verbose_router_logger.error( "litellm.router.py::async_function_with_fallbacks() - " - "Error occurred while trying to do fallbacks - %s\n%s\n" + "Error occurred while trying to do fallbacks - %s\n" "Debug Information:\nCooldown Deployments=%s", fallback_failure_exception_str, - redact_string(traceback.format_exc()), cooldown_info, ) @@ -7162,7 +7168,7 @@ class Router: k, v, ) in kwargs.items(): # log everything in kwargs except the old previous_models value - prevent nesting - if k not in [_metadata_var, "messages", "original_function"]: + if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS: previous_model[k] = v elif k == _metadata_var and isinstance(v, dict): previous_model[_metadata_var] = {} diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 1c628ef9ee3..6fad2dd31e9 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -7,6 +7,7 @@ if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIFileObject from litellm._logging import verbose_logger +from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError from litellm.types.router import CredentialLiteLLMParams @@ -43,6 +44,22 @@ def resolve_model_group_alias(model_group_alias: object, model: str) -> str | No return target +def truncate_fallback_error_detail(detail: str) -> str: + """ + Bound a fallback failure detail before it is logged or appended to an exception message. + + Each level of the fallback walk records the failure of the level below it, so an + untruncated detail carries every nested failure with it and grows superlinearly with + the number of attempted model groups. One deterministic pre-network failure walked + through a small fallback graph is enough to turn that into hundreds of megabytes of + output on the event-loop thread, which starves the process that produced it. + """ + if len(detail) <= ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: + return detail + dropped: Final = len(detail) - ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS + return f"{detail[:ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS]}... [truncated {dropped} characters]" + + def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str: """ Hash of the credential params, used for mapping the file id to the right model diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 1c6bb52ccb8..ef48ccc821e 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -1,3 +1,6 @@ +import hashlib +import json +from dataclasses import dataclass from enum import Enum from typing import TYPE_CHECKING, Any, Final @@ -19,6 +22,52 @@ else: LitellmRouter = Any +def fallback_attempt_key(fallback_target: object) -> str | None: + """ + Identity of one fallback attempt, so the same attempt is never made twice per request. + + A bare model group name and a `{"model": name}` entry describe the same attempt. An + entry carrying anything else describes a different one and keeps its own identity: a + client-side fallback list overrides request params such as `messages`, and the router + re-targets the group that just failed by attaching `_target_order` or + `_excluded_deployment_ids` to select a different set of deployments inside it. The + payload is hashed rather than kept, so a large `messages` override does not make the + request hold a second copy of itself. + + Returns None for a shape with no usable identity, which is never skipped. + """ + if isinstance(fallback_target, str): + return fallback_target + if not isinstance(fallback_target, dict): + return None + model: Final = fallback_target.get("model") + if tuple(fallback_target) == ("model",) and isinstance(model, str): + return model + serialized: Final = json.dumps(fallback_target, sort_keys=True, default=str) + return hashlib.sha256(serialized.encode()).hexdigest() + + +@dataclass(slots=True) +class AttemptedFallbackTargets: + """ + The fallback attempts a single request has already made. + + One instance is created on the first fallback hop and shared by reference for the rest + of the walk, so an attempt made in one branch is not repeated in a sibling branch. + Without it the walk enumerates paths rather than attempts: a fallback graph containing + a cycle retries one deterministic failure once per path through the cycle, and a + client-side fallback list is re-walked at every level of the recursion. + """ + + keys: frozenset[str] = frozenset() + + def __contains__(self, key: str) -> bool: + return key in self.keys + + def record(self, key: str) -> None: + self.keys = self.keys | frozenset((key,)) + + def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: """ Handles wildcard routing scenario @@ -106,7 +155,14 @@ async def run_async_fallback( fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"] original_model_group: The original model group. example: "gpt-3.5-turbo" original_exception: The original exception. - **kwargs: Keyword arguments. + **kwargs: Keyword arguments. `attempted_targets` carries the fallback attempts + already made for this request, created on the first hop and shared by reference + for the rest of the walk. A target already in it is skipped, so neither a + fallback graph that loops back on itself nor a client-side fallback list + re-walked at each level can repeat an attempt that has already failed. Identity + comes from `fallback_attempt_key`, so an entry that overrides request params or + re-targets the failed group with a different deployment selection stays distinct + from a bare name. Returns: The response from the successful fallback model group. @@ -120,10 +176,27 @@ async def run_async_fallback( error_from_fallbacks = original_exception fallback_errors = (get_fallback_error_info(original_exception),) + # Read out of kwargs and narrowed here rather than declared as a parameter: every caller + # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter + # would carry an annotation that no call site can actually be checked against. + carried_targets: Final = kwargs.get("attempted_targets") + attempted: Final = ( + carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets() + ) + attempted.record(original_model_group) for mg in fallback_model_group: if mg == original_model_group: continue + attempt_key = fallback_attempt_key(mg) + if attempt_key is not None: + if attempt_key in attempted: + verbose_router_logger.info( + "Skipping fallback to model_group = %s, already attempted for this request", + mask_sensitive_structure(mg), + ) + continue + attempted.record(attempt_key) try: # LOGGING kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception) @@ -138,6 +211,7 @@ async def run_async_fallback( fallback_depth = fallback_depth + 1 kwargs["fallback_depth"] = fallback_depth kwargs["max_fallbacks"] = max_fallbacks + kwargs["attempted_targets"] = attempted if include_fallback_errors: kwargs["include_fallback_errors"] = include_fallback_errors response = await litellm_router.async_function_with_fallbacks(*args, **kwargs) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 329b10e72e7..35d4250782f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3479,6 +3479,7 @@ all_litellm_params = ( "user_continue_message", "fallback_depth", "max_fallbacks", + "attempted_targets", "max_budget", "budget_duration", "use_in_pass_through", diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 98a34de295c..e2348d28701 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -3,6 +3,8 @@ import json import pytest from litellm.router_utils.fallback_event_handlers import ( + AttemptedFallbackTargets, + fallback_attempt_key, get_fallback_model_group, run_async_fallback, ) @@ -142,6 +144,208 @@ async def test_run_async_fallback_skips_original_model_group(): assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 +class RecordingFailRouter: + def __init__(self): + self.attempted_models = [] + + def log_retry(self, kwargs, e): + return kwargs + + async def async_function_with_fallbacks(self, *args, **kwargs): + self.attempted_models.append(kwargs.get("model")) + raise RuntimeError("fallback model also failed") + + +@pytest.mark.asyncio +async def test_run_async_fallback_skips_model_group_already_attempted(): + """A fallback graph that loops back on itself must not re-attempt a model group that + already failed for this request. Every group in a cycle fails identically, so + revisiting one multiplies the work and the error output without any chance of + succeeding.""" + router = RecordingFailRouter() + + with pytest.raises(RuntimeError, match="original failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["already-attempted"], + original_model_group="primary-model", + original_exception=RuntimeError("original failed"), + max_fallbacks=3, + fallback_depth=0, + attempted_targets=AttemptedFallbackTargets(frozenset({"already-attempted"})), + ) + + assert router.attempted_models == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_attempts_a_repeated_target_once(): + router = RecordingFailRouter() + + with pytest.raises(RuntimeError, match="fallback model also failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model", "fallback-model", "other-model"], + original_model_group="primary-model", + original_exception=RuntimeError("original failed"), + max_fallbacks=5, + fallback_depth=0, + ) + + assert router.attempted_models == ["fallback-model", "other-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_forwards_attempted_model_groups_to_nested_call(): + """The nested call is where the next hop of the walk decides what to skip, so the + accumulated set has to reach it, carrying both the group that just failed and the + target being attempted.""" + router = RecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("original failed"), + max_fallbacks=3, + fallback_depth=0, + attempted_targets=AttemptedFallbackTargets(frozenset({"earlier-model"})), + ) + + assert router.received_kwargs["attempted_targets"].keys == frozenset( + {"earlier-model", "primary-model", "fallback-model"} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "entry", + [ + {"model": "primary-model", "_target_order": 2}, + {"model": "primary-model", "_excluded_deployment_ids": ["dep-1"]}, + ], +) +async def test_run_async_fallback_still_retargets_the_same_group_via_dict_entry(entry): + """Order-based fallback and weighted intra-group failover both re-target the group that + just failed, selecting a different set of deployments inside it. Those entries are dicts + rather than plain names and must survive a guard that skips already-attempted names.""" + router = RecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=[entry], + original_model_group="primary-model", + original_exception=RuntimeError("original failed"), + max_fallbacks=3, + fallback_depth=0, + attempted_targets=AttemptedFallbackTargets(frozenset({"primary-model"})), + ) + + assert router.received_kwargs["model"] == "primary-model" + + +@pytest.mark.asyncio +async def test_run_async_fallback_skips_a_repeated_dict_target(): + """A client-side fallback list names its targets with dicts, and that list is re-walked + at every level of the recursion, so an entry that carries no request override has to be + recognised as the same attempt as the bare name.""" + router = RecordingFailRouter() + + with pytest.raises(RuntimeError, match="original failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=[{"model": "already-attempted"}], + original_model_group="primary-model", + original_exception=RuntimeError("original failed"), + max_fallbacks=3, + fallback_depth=0, + attempted_targets=AttemptedFallbackTargets(frozenset({"already-attempted"})), + ) + + assert router.attempted_models == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_attempts_a_repeated_dict_target_once(): + router = RecordingFailRouter() + entry = {"model": "fallback-model", "messages": [{"role": "user", "content": "shorter"}]} + + with pytest.raises(RuntimeError, match="fallback model also failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=[entry, entry, {"model": "other-model"}], + original_model_group="primary-model", + original_exception=RuntimeError("original failed"), + max_fallbacks=5, + fallback_depth=0, + ) + + assert router.attempted_models == ["fallback-model", "other-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_keeps_a_request_override_distinct_from_the_bare_name(): + """The documented use of the client-side form is to retry a group with different request + params, so an entry carrying an override must survive even when the bare name of that + same group has already been attempted.""" + router = RecordingFailRouter() + + with pytest.raises(RuntimeError, match="fallback model also failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=[ + {"model": "already-attempted", "messages": [{"role": "user", "content": "shorter"}]} + ], + original_model_group="primary-model", + original_exception=RuntimeError("original failed"), + max_fallbacks=3, + fallback_depth=0, + attempted_targets=AttemptedFallbackTargets(frozenset({"already-attempted"})), + ) + + assert router.attempted_models == ["already-attempted"] + + +@pytest.mark.parametrize( + "target, expected", + [ + ("group-a", "group-a"), + ({"model": "group-a"}, "group-a"), + (None, None), + (["group-a"], None), + ], +) +def test_fallback_attempt_key_identity(target, expected): + """A bare name and a `{"model": name}` entry are the same attempt. A shape with no + usable identity returns None and is never skipped, so an unrecognised entry keeps + today's behaviour rather than being silently dropped.""" + assert fallback_attempt_key(target) == expected + + +def test_fallback_attempt_key_gives_a_param_only_entry_its_own_identity(): + """An entry with no `model` re-targets the group currently being attempted with + different request params, so it is a distinct attempt and still needs an identity.""" + key = fallback_attempt_key({"messages": [{"role": "user", "content": "shorter"}]}) + + assert key is not None + assert key != fallback_attempt_key({"messages": [{"role": "user", "content": "other"}]}) + + +def test_fallback_attempt_key_separates_overrides_from_the_bare_name(): + bare = fallback_attempt_key("group-a") + override = fallback_attempt_key({"model": "group-a", "messages": [{"role": "user", "content": "x"}]}) + other_override = fallback_attempt_key({"model": "group-a", "messages": [{"role": "user", "content": "y"}]}) + order_retarget = fallback_attempt_key({"model": "group-a", "_target_order": 2}) + + assert len({bare, override, other_override, order_retarget}) == 4 + + +def test_fallback_attempt_key_is_stable_across_key_order(): + assert fallback_attempt_key({"model": "group-a", "_target_order": 2}) == fallback_attempt_key( + {"_target_order": 2, "model": "group-a"} + ) + + def test_get_fallback_model_group_does_not_mutate_fallbacks(): """A string fallback must be resolved without mutating the caller's fallbacks list, which is the live router config shared across requests.""" diff --git a/tests/test_litellm/router_utils/test_router_utils_common_utils.py b/tests/test_litellm/router_utils/test_router_utils_common_utils.py index 7d453c72652..0d063ad14f5 100644 --- a/tests/test_litellm/router_utils/test_router_utils_common_utils.py +++ b/tests/test_litellm/router_utils/test_router_utils_common_utils.py @@ -4,6 +4,7 @@ from unittest.mock import Mock import pytest from litellm import Router +from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.common_utils import ( _deployment_supports_web_search, @@ -11,6 +12,7 @@ from litellm.router_utils.common_utils import ( filter_team_based_models, filter_web_search_deployments, resolve_model_group_alias, + truncate_fallback_error_detail, ) @@ -558,3 +560,27 @@ class TestResolveModelGroupAlias: assert router._get_model_from_alias("group-a") == "group-b" assert router._get_model_from_alias("group-item") == "group-b" assert router._get_model_from_alias("group-b") is None + + +class TestTruncateFallbackErrorDetail: + def test_short_detail_is_returned_unchanged(self): + assert truncate_fallback_error_detail("boom") == "boom" + + def test_detail_at_the_limit_is_returned_unchanged(self): + detail = "x" * ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS + assert truncate_fallback_error_detail(detail) == detail + + def test_long_detail_is_bounded_and_reports_what_was_dropped(self): + detail = "x" * (ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS + 500) + + truncated = truncate_fallback_error_detail(detail) + + assert truncated.startswith("x" * ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS) + assert truncated.endswith("... [truncated 500 characters]") + assert len(truncated) < len(detail) + + def test_a_megabyte_of_detail_comes_back_small(self): + """The detail is what a fallback level records about the level below it, so it has + to stay small enough that a walk over many model groups cannot compound it into an + output volume that starves the process.""" + assert len(truncate_fallback_error_detail("x" * 1_000_000)) < 3_000 diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 4a3395a7d3f..da2d78edb73 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1,6 +1,7 @@ import asyncio import copy import json +import logging import os import sys from unittest.mock import AsyncMock, MagicMock, patch @@ -14,6 +15,7 @@ sys.path.insert( import litellm from litellm.exceptions import MidStreamFallbackError +from litellm.integrations.custom_logger import CustomLogger def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): @@ -7415,3 +7417,136 @@ class TestAutoRouterMaxInputCharsWiring: router = self._router() assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS + + +class _LogCapture(logging.Handler): + def __init__(self, level): + super().__init__(level=level) + self._level = level + self.messages = [] + + def emit(self, record): + if record.levelno == self._level: + self.messages.append(record.getMessage()) + + +class _FallbackAttemptRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.failed_targets = [] + + async def log_failure_fallback_event(self, original_model_group, kwargs, original_exception): + self.failed_targets.append(kwargs.get("model")) + + +def _cyclic_fallback_router(num_retries=0): + groups = ["group-a", "group-b", "group-c", "group-d"] + return litellm.Router( + model_list=[ + { + "model_name": group, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-fake", + "mock_response": "litellm.InternalServerError", + }, + } + for group in groups + ], + fallbacks=[ + {"group-a": ["group-b", "group-c"]}, + {"group-b": ["group-a", "group-c"]}, + {"group-c": ["group-d"]}, + {"group-d": ["group-b", "group-a"]}, + ], + num_retries=num_retries, + ) + + +async def _drive_cyclic_fallback(router, capture, recorder=None, **request_kwargs): + router_logger = logging.getLogger("LiteLLM Router") + previous_level = router_logger.level + router_logger.setLevel(capture.level) + router_logger.addHandler(capture) + if recorder is not None: + litellm.callbacks.append(recorder) + try: + with pytest.raises(litellm.InternalServerError): + await router.acompletion( + model="group-a", messages=[{"role": "user", "content": "hi"}], **request_kwargs + ) + finally: + router_logger.removeHandler(capture) + router_logger.setLevel(previous_level) + if recorder is not None: + litellm.callbacks.remove(recorder) + + +@pytest.mark.asyncio +async def test_cyclic_fallback_graph_does_not_amplify_one_request(): + """A fallback graph whose entries loop back on each other is easy to build by accident, + and every group in the loop fails identically on a deterministic error, so the walk must + not revisit a group and must not re-emit a growing chained traceback at each level. Left + unbounded, one request blocks the event loop long enough for health probes to fail.""" + recorder = _FallbackAttemptRecorder() + capture = _LogCapture(logging.ERROR) + + await _drive_cyclic_fallback(_cyclic_fallback_router(), capture, recorder) + + assert sorted(set(recorder.failed_targets)) == ["group-b", "group-c", "group-d"] + assert len(recorder.failed_targets) == len(set(recorder.failed_targets)) + assert not any("Traceback (most recent call last)" in message for message in capture.messages) + assert sum(len(message) for message in capture.messages) < 5_000 + + +@pytest.mark.asyncio +async def test_retry_breadcrumbs_do_not_carry_the_walk_state(): + """log_retry copies every kwarg into previous_models, which reaches spend logs and + logging callbacks. The set of already-attempted groups is router-internal walk state + with no diagnostic value there, and it is the one entry that is not a plain scalar. + A retry has to be configured for the walk state to reach log_retry at all.""" + router = _cyclic_fallback_router(num_retries=1) + capture = _LogCapture(logging.ERROR) + + await _drive_cyclic_fallback(router, capture) + + assert router.previous_models, "no retry breadcrumbs were recorded" + assert any( + "fallback_depth" in breadcrumb for breadcrumb in router.previous_models + ), "no breadcrumb carried router walk state, so this test cannot see the leak" + for breadcrumb in router.previous_models: + assert "attempted_targets" not in breadcrumb + + +@pytest.mark.asyncio +async def test_fallback_traceback_stays_available_at_debug_level(): + """Dropping the stack from the ERROR line is only safe because the fallback path still + emits it once per level at DEBUG, which is what an operator needs to diagnose why every + fallback failed. This pins that remaining debug traceback.""" + capture = _LogCapture(logging.DEBUG) + + await _drive_cyclic_fallback(_cyclic_fallback_router(), capture) + + assert any("Traceback (most recent call last)" in message for message in capture.messages) + + +@pytest.mark.asyncio +async def test_fallback_failure_detail_from_upstream_is_bounded(): + """The detail each level records about the level below it is attacker-influenced, since + it carries whatever the upstream error said. It has to be bounded on its own, so a walk + over several groups cannot compound one large message into the log or into the message + handed back to the caller.""" + huge_message = "z" * 50_000 + capture = _LogCapture(logging.ERROR) + + await _drive_cyclic_fallback( + _cyclic_fallback_router(), + capture, + mock_response=litellm.InternalServerError( + message=huge_message, llm_provider="openai", model="group-a" + ), + ) + + assert capture.messages, "the fallback failure path did not log at ERROR" + assert huge_message not in "".join(capture.messages) + assert max(len(message) for message in capture.messages) < 5_000