fix(router): bound fallback-walk work and error-log volume (#36148)

litellm/constants.py

@@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_non
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
 ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
+ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
 DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
 DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10))

litellm/router.py

@@ -109,6 +109,7 @@ from litellm.router_utils.common_utils import (
     filter_team_based_models,
     filter_web_search_deployments,
     resolve_model_group_alias,
+    truncate_fallback_error_detail,
 )
 from litellm.router_utils.cooldown_cache import CooldownCache
 from litellm.router_utils.cooldown_handlers import (
@@ -342,6 +343,12 @@ def _replay_live_router_model_cost() -> None:
 set_live_deployment_replay(_replay_live_router_model_cost)
 
 
+# Kwargs that log_retry must not copy into a retry breadcrumb. The breadcrumbs reach spend
+# logs and logging callbacks, and these carry either the request payload or router-internal
+# walk state rather than anything that identifies the failed attempt.
+RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset(("messages", "original_function", "attempted_targets"))
+
+
 class Router:
     model_names: set = set()
     cache_responses: bool | None = False
@@ -6361,17 +6368,16 @@ class Router:
                 return response
         except Exception as new_exception:
             parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
-            fallback_failure_exception_str = redact_string(str(new_exception))
+            fallback_failure_exception_str = truncate_fallback_error_detail(redact_string(str(new_exception)))
             cooldown_info: Final = await _async_get_cooldown_deployments_with_debug_info(
                 litellm_router_instance=self,
                 parent_otel_span=parent_otel_span,
             )
             verbose_router_logger.error(
                 "litellm.router.py::async_function_with_fallbacks() - "
-                "Error occurred while trying to do fallbacks - %s\n%s\n"
+                "Error occurred while trying to do fallbacks - %s\n"
                 "Debug Information:\nCooldown Deployments=%s",
                 fallback_failure_exception_str,
-                redact_string(traceback.format_exc()),
                 cooldown_info,
             )
 
@@ -7162,7 +7168,7 @@ class Router:
                 k,
                 v,
             ) in kwargs.items():  # log everything in kwargs except the old previous_models value - prevent nesting
-                if k not in [_metadata_var, "messages", "original_function"]:
+                if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS:
                     previous_model[k] = v
                 elif k == _metadata_var and isinstance(v, dict):
                     previous_model[_metadata_var] = {}

litellm/router_utils/common_utils.py

@@ -7,6 +7,7 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import OpenAIFileObject
 
 from litellm._logging import verbose_logger
+from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
 from litellm.exceptions import BadRequestError
 from litellm.types.router import CredentialLiteLLMParams
 
@@ -43,6 +44,22 @@ def resolve_model_group_alias(model_group_alias: object, model: str) -> str | No
     return target
 
 
+def truncate_fallback_error_detail(detail: str) -> str:
+    """
+    Bound a fallback failure detail before it is logged or appended to an exception message.
+
+    Each level of the fallback walk records the failure of the level below it, so an
+    untruncated detail carries every nested failure with it and grows superlinearly with
+    the number of attempted model groups. One deterministic pre-network failure walked
+    through a small fallback graph is enough to turn that into hundreds of megabytes of
+    output on the event-loop thread, which starves the process that produced it.
+    """
+    if len(detail) <= ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS:
+        return detail
+    dropped: Final = len(detail) - ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
+    return f"{detail[:ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS]}... [truncated {dropped} characters]"
+
+
 def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:
     """
     Hash of the credential params, used for mapping the file id to the right model

litellm/router_utils/fallback_event_handlers.py

@@ -1,3 +1,6 @@
+import hashlib
+import json
+from dataclasses import dataclass
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Final
 
@@ -19,6 +22,52 @@ else:
     LitellmRouter = Any
 
 
+def fallback_attempt_key(fallback_target: object) -> str | None:
+    """
+    Identity of one fallback attempt, so the same attempt is never made twice per request.
+
+    A bare model group name and a `{"model": name}` entry describe the same attempt. An
+    entry carrying anything else describes a different one and keeps its own identity: a
+    client-side fallback list overrides request params such as `messages`, and the router
+    re-targets the group that just failed by attaching `_target_order` or
+    `_excluded_deployment_ids` to select a different set of deployments inside it. The
+    payload is hashed rather than kept, so a large `messages` override does not make the
+    request hold a second copy of itself.
+
+    Returns None for a shape with no usable identity, which is never skipped.
+    """
+    if isinstance(fallback_target, str):
+        return fallback_target
+    if not isinstance(fallback_target, dict):
+        return None
+    model: Final = fallback_target.get("model")
+    if tuple(fallback_target) == ("model",) and isinstance(model, str):
+        return model
+    serialized: Final = json.dumps(fallback_target, sort_keys=True, default=str)
+    return hashlib.sha256(serialized.encode()).hexdigest()
+
+
+@dataclass(slots=True)
+class AttemptedFallbackTargets:
+    """
+    The fallback attempts a single request has already made.
+
+    One instance is created on the first fallback hop and shared by reference for the rest
+    of the walk, so an attempt made in one branch is not repeated in a sibling branch.
+    Without it the walk enumerates paths rather than attempts: a fallback graph containing
+    a cycle retries one deterministic failure once per path through the cycle, and a
+    client-side fallback list is re-walked at every level of the recursion.
+    """
+
+    keys: frozenset[str] = frozenset()
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.keys
+
+    def record(self, key: str) -> None:
+        self.keys = self.keys | frozenset((key,))
+
+
 def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool:
     """
     Handles wildcard routing scenario
@@ -106,7 +155,14 @@ async def run_async_fallback(
         fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"]
         original_model_group: The original model group. example: "gpt-3.5-turbo"
         original_exception: The original exception.
-        **kwargs: Keyword arguments.
+        **kwargs: Keyword arguments. `attempted_targets` carries the fallback attempts
+            already made for this request, created on the first hop and shared by reference
+            for the rest of the walk. A target already in it is skipped, so neither a
+            fallback graph that loops back on itself nor a client-side fallback list
+            re-walked at each level can repeat an attempt that has already failed. Identity
+            comes from `fallback_attempt_key`, so an entry that overrides request params or
+            re-targets the failed group with a different deployment selection stays distinct
+            from a bare name.
 
     Returns:
         The response from the successful fallback model group.
@@ -120,10 +176,27 @@ async def run_async_fallback(
 
     error_from_fallbacks = original_exception
     fallback_errors = (get_fallback_error_info(original_exception),)
+    # Read out of kwargs and narrowed here rather than declared as a parameter: every caller
+    # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter
+    # would carry an annotation that no call site can actually be checked against.
+    carried_targets: Final = kwargs.get("attempted_targets")
+    attempted: Final = (
+        carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets()
+    )
+    attempted.record(original_model_group)
 
     for mg in fallback_model_group:
         if mg == original_model_group:
             continue
+        attempt_key = fallback_attempt_key(mg)
+        if attempt_key is not None:
+            if attempt_key in attempted:
+                verbose_router_logger.info(
+                    "Skipping fallback to model_group = %s, already attempted for this request",
+                    mask_sensitive_structure(mg),
+                )
+                continue
+            attempted.record(attempt_key)
         try:
             # LOGGING
             kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
@@ -138,6 +211,7 @@ async def run_async_fallback(
             fallback_depth = fallback_depth + 1
             kwargs["fallback_depth"] = fallback_depth
             kwargs["max_fallbacks"] = max_fallbacks
+            kwargs["attempted_targets"] = attempted
             if include_fallback_errors:
                 kwargs["include_fallback_errors"] = include_fallback_errors
             response = await litellm_router.async_function_with_fallbacks(*args, **kwargs)

litellm/types/utils.py

@@ -3479,6 +3479,7 @@ all_litellm_params = (
         "user_continue_message",
         "fallback_depth",
         "max_fallbacks",
+        "attempted_targets",
         "max_budget",
         "budget_duration",
         "use_in_pass_through",

tests/test_litellm/router_utils/test_fallback_event_handlers.py

@@ -3,6 +3,8 @@ import json
 import pytest
 
 from litellm.router_utils.fallback_event_handlers import (
+    AttemptedFallbackTargets,
+    fallback_attempt_key,
     get_fallback_model_group,
     run_async_fallback,
 )
@@ -142,6 +144,208 @@ async def test_run_async_fallback_skips_original_model_group():
     assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1
 
 
+class RecordingFailRouter:
+    def __init__(self):
+        self.attempted_models = []
+
+    def log_retry(self, kwargs, e):
+        return kwargs
+
+    async def async_function_with_fallbacks(self, *args, **kwargs):
+        self.attempted_models.append(kwargs.get("model"))
+        raise RuntimeError("fallback model also failed")
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_skips_model_group_already_attempted():
+    """A fallback graph that loops back on itself must not re-attempt a model group that
+    already failed for this request. Every group in a cycle fails identically, so
+    revisiting one multiplies the work and the error output without any chance of
+    succeeding."""
+    router = RecordingFailRouter()
+
+    with pytest.raises(RuntimeError, match="original failed"):
+        await run_async_fallback(
+            litellm_router=router,
+            fallback_model_group=["already-attempted"],
+            original_model_group="primary-model",
+            original_exception=RuntimeError("original failed"),
+            max_fallbacks=3,
+            fallback_depth=0,
+            attempted_targets=AttemptedFallbackTargets(frozenset({"already-attempted"})),
+        )
+
+    assert router.attempted_models == []
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_attempts_a_repeated_target_once():
+    router = RecordingFailRouter()
+
+    with pytest.raises(RuntimeError, match="fallback model also failed"):
+        await run_async_fallback(
+            litellm_router=router,
+            fallback_model_group=["fallback-model", "fallback-model", "other-model"],
+            original_model_group="primary-model",
+            original_exception=RuntimeError("original failed"),
+            max_fallbacks=5,
+            fallback_depth=0,
+        )
+
+    assert router.attempted_models == ["fallback-model", "other-model"]
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_forwards_attempted_model_groups_to_nested_call():
+    """The nested call is where the next hop of the walk decides what to skip, so the
+    accumulated set has to reach it, carrying both the group that just failed and the
+    target being attempted."""
+    router = RecordingRouter()
+
+    await run_async_fallback(
+        litellm_router=router,
+        fallback_model_group=["fallback-model"],
+        original_model_group="primary-model",
+        original_exception=RuntimeError("original failed"),
+        max_fallbacks=3,
+        fallback_depth=0,
+        attempted_targets=AttemptedFallbackTargets(frozenset({"earlier-model"})),
+    )
+
+    assert router.received_kwargs["attempted_targets"].keys == frozenset(
+        {"earlier-model", "primary-model", "fallback-model"}
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "entry",
+    [
+        {"model": "primary-model", "_target_order": 2},
+        {"model": "primary-model", "_excluded_deployment_ids": ["dep-1"]},
+    ],
+)
+async def test_run_async_fallback_still_retargets_the_same_group_via_dict_entry(entry):
+    """Order-based fallback and weighted intra-group failover both re-target the group that
+    just failed, selecting a different set of deployments inside it. Those entries are dicts
+    rather than plain names and must survive a guard that skips already-attempted names."""
+    router = RecordingRouter()
+
+    await run_async_fallback(
+        litellm_router=router,
+        fallback_model_group=[entry],
+        original_model_group="primary-model",
+        original_exception=RuntimeError("original failed"),
+        max_fallbacks=3,
+        fallback_depth=0,
+        attempted_targets=AttemptedFallbackTargets(frozenset({"primary-model"})),
+    )
+
+    assert router.received_kwargs["model"] == "primary-model"
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_skips_a_repeated_dict_target():
+    """A client-side fallback list names its targets with dicts, and that list is re-walked
+    at every level of the recursion, so an entry that carries no request override has to be
+    recognised as the same attempt as the bare name."""
+    router = RecordingFailRouter()
+
+    with pytest.raises(RuntimeError, match="original failed"):
+        await run_async_fallback(
+            litellm_router=router,
+            fallback_model_group=[{"model": "already-attempted"}],
+            original_model_group="primary-model",
+            original_exception=RuntimeError("original failed"),
+            max_fallbacks=3,
+            fallback_depth=0,
+            attempted_targets=AttemptedFallbackTargets(frozenset({"already-attempted"})),
+        )
+
+    assert router.attempted_models == []
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_attempts_a_repeated_dict_target_once():
+    router = RecordingFailRouter()
+    entry = {"model": "fallback-model", "messages": [{"role": "user", "content": "shorter"}]}
+
+    with pytest.raises(RuntimeError, match="fallback model also failed"):
+        await run_async_fallback(
+            litellm_router=router,
+            fallback_model_group=[entry, entry, {"model": "other-model"}],
+            original_model_group="primary-model",
+            original_exception=RuntimeError("original failed"),
+            max_fallbacks=5,
+            fallback_depth=0,
+        )
+
+    assert router.attempted_models == ["fallback-model", "other-model"]
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_keeps_a_request_override_distinct_from_the_bare_name():
+    """The documented use of the client-side form is to retry a group with different request
+    params, so an entry carrying an override must survive even when the bare name of that
+    same group has already been attempted."""
+    router = RecordingFailRouter()
+
+    with pytest.raises(RuntimeError, match="fallback model also failed"):
+        await run_async_fallback(
+            litellm_router=router,
+            fallback_model_group=[
+                {"model": "already-attempted", "messages": [{"role": "user", "content": "shorter"}]}
+            ],
+            original_model_group="primary-model",
+            original_exception=RuntimeError("original failed"),
+            max_fallbacks=3,
+            fallback_depth=0,
+            attempted_targets=AttemptedFallbackTargets(frozenset({"already-attempted"})),
+        )
+
+    assert router.attempted_models == ["already-attempted"]
+
+
+@pytest.mark.parametrize(
+    "target, expected",
+    [
+        ("group-a", "group-a"),
+        ({"model": "group-a"}, "group-a"),
+        (None, None),
+        (["group-a"], None),
+    ],
+)
+def test_fallback_attempt_key_identity(target, expected):
+    """A bare name and a `{"model": name}` entry are the same attempt. A shape with no
+    usable identity returns None and is never skipped, so an unrecognised entry keeps
+    today's behaviour rather than being silently dropped."""
+    assert fallback_attempt_key(target) == expected
+
+
+def test_fallback_attempt_key_gives_a_param_only_entry_its_own_identity():
+    """An entry with no `model` re-targets the group currently being attempted with
+    different request params, so it is a distinct attempt and still needs an identity."""
+    key = fallback_attempt_key({"messages": [{"role": "user", "content": "shorter"}]})
+
+    assert key is not None
+    assert key != fallback_attempt_key({"messages": [{"role": "user", "content": "other"}]})
+
+
+def test_fallback_attempt_key_separates_overrides_from_the_bare_name():
+    bare = fallback_attempt_key("group-a")
+    override = fallback_attempt_key({"model": "group-a", "messages": [{"role": "user", "content": "x"}]})
+    other_override = fallback_attempt_key({"model": "group-a", "messages": [{"role": "user", "content": "y"}]})
+    order_retarget = fallback_attempt_key({"model": "group-a", "_target_order": 2})
+
+    assert len({bare, override, other_override, order_retarget}) == 4
+
+
+def test_fallback_attempt_key_is_stable_across_key_order():
+    assert fallback_attempt_key({"model": "group-a", "_target_order": 2}) == fallback_attempt_key(
+        {"_target_order": 2, "model": "group-a"}
+    )
+
+
 def test_get_fallback_model_group_does_not_mutate_fallbacks():
     """A string fallback must be resolved without mutating the caller's
     fallbacks list, which is the live router config shared across requests."""

tests/test_litellm/router_utils/test_router_utils_common_utils.py

@@ -4,6 +4,7 @@ from unittest.mock import Mock
 import pytest
 
 from litellm import Router
+from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.router_utils.common_utils import (
     _deployment_supports_web_search,
@@ -11,6 +12,7 @@ from litellm.router_utils.common_utils import (
     filter_team_based_models,
     filter_web_search_deployments,
     resolve_model_group_alias,
+    truncate_fallback_error_detail,
 )
 
 
@@ -558,3 +560,27 @@ class TestResolveModelGroupAlias:
         assert router._get_model_from_alias("group-a") == "group-b"
         assert router._get_model_from_alias("group-item") == "group-b"
         assert router._get_model_from_alias("group-b") is None
+
+
+class TestTruncateFallbackErrorDetail:
+    def test_short_detail_is_returned_unchanged(self):
+        assert truncate_fallback_error_detail("boom") == "boom"
+
+    def test_detail_at_the_limit_is_returned_unchanged(self):
+        detail = "x" * ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
+        assert truncate_fallback_error_detail(detail) == detail
+
+    def test_long_detail_is_bounded_and_reports_what_was_dropped(self):
+        detail = "x" * (ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS + 500)
+
+        truncated = truncate_fallback_error_detail(detail)
+
+        assert truncated.startswith("x" * ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS)
+        assert truncated.endswith("... [truncated 500 characters]")
+        assert len(truncated) < len(detail)
+
+    def test_a_megabyte_of_detail_comes_back_small(self):
+        """The detail is what a fallback level records about the level below it, so it has
+        to stay small enough that a walk over many model groups cannot compound it into an
+        output volume that starves the process."""
+        assert len(truncate_fallback_error_detail("x" * 1_000_000)) < 3_000

tests/test_litellm/test_router.py

@@ -1,6 +1,7 @@
 import asyncio
 import copy
 import json
+import logging
 import os
 import sys
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -14,6 +15,7 @@ sys.path.insert(
 
 import litellm
 from litellm.exceptions import MidStreamFallbackError
+from litellm.integrations.custom_logger import CustomLogger
 
 
 def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
@@ -7415,3 +7417,136 @@ class TestAutoRouterMaxInputCharsWiring:
         router = self._router()
 
         assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
+
+
+class _LogCapture(logging.Handler):
+    def __init__(self, level):
+        super().__init__(level=level)
+        self._level = level
+        self.messages = []
+
+    def emit(self, record):
+        if record.levelno == self._level:
+            self.messages.append(record.getMessage())
+
+
+class _FallbackAttemptRecorder(CustomLogger):
+    def __init__(self):
+        super().__init__()
+        self.failed_targets = []
+
+    async def log_failure_fallback_event(self, original_model_group, kwargs, original_exception):
+        self.failed_targets.append(kwargs.get("model"))
+
+
+def _cyclic_fallback_router(num_retries=0):
+    groups = ["group-a", "group-b", "group-c", "group-d"]
+    return litellm.Router(
+        model_list=[
+            {
+                "model_name": group,
+                "litellm_params": {
+                    "model": "openai/gpt-4o-mini",
+                    "api_key": "sk-fake",
+                    "mock_response": "litellm.InternalServerError",
+                },
+            }
+            for group in groups
+        ],
+        fallbacks=[
+            {"group-a": ["group-b", "group-c"]},
+            {"group-b": ["group-a", "group-c"]},
+            {"group-c": ["group-d"]},
+            {"group-d": ["group-b", "group-a"]},
+        ],
+        num_retries=num_retries,
+    )
+
+
+async def _drive_cyclic_fallback(router, capture, recorder=None, **request_kwargs):
+    router_logger = logging.getLogger("LiteLLM Router")
+    previous_level = router_logger.level
+    router_logger.setLevel(capture.level)
+    router_logger.addHandler(capture)
+    if recorder is not None:
+        litellm.callbacks.append(recorder)
+    try:
+        with pytest.raises(litellm.InternalServerError):
+            await router.acompletion(
+                model="group-a", messages=[{"role": "user", "content": "hi"}], **request_kwargs
+            )
+    finally:
+        router_logger.removeHandler(capture)
+        router_logger.setLevel(previous_level)
+        if recorder is not None:
+            litellm.callbacks.remove(recorder)
+
+
+@pytest.mark.asyncio
+async def test_cyclic_fallback_graph_does_not_amplify_one_request():
+    """A fallback graph whose entries loop back on each other is easy to build by accident,
+    and every group in the loop fails identically on a deterministic error, so the walk must
+    not revisit a group and must not re-emit a growing chained traceback at each level. Left
+    unbounded, one request blocks the event loop long enough for health probes to fail."""
+    recorder = _FallbackAttemptRecorder()
+    capture = _LogCapture(logging.ERROR)
+
+    await _drive_cyclic_fallback(_cyclic_fallback_router(), capture, recorder)
+
+    assert sorted(set(recorder.failed_targets)) == ["group-b", "group-c", "group-d"]
+    assert len(recorder.failed_targets) == len(set(recorder.failed_targets))
+    assert not any("Traceback (most recent call last)" in message for message in capture.messages)
+    assert sum(len(message) for message in capture.messages) < 5_000
+
+
+@pytest.mark.asyncio
+async def test_retry_breadcrumbs_do_not_carry_the_walk_state():
+    """log_retry copies every kwarg into previous_models, which reaches spend logs and
+    logging callbacks. The set of already-attempted groups is router-internal walk state
+    with no diagnostic value there, and it is the one entry that is not a plain scalar.
+    A retry has to be configured for the walk state to reach log_retry at all."""
+    router = _cyclic_fallback_router(num_retries=1)
+    capture = _LogCapture(logging.ERROR)
+
+    await _drive_cyclic_fallback(router, capture)
+
+    assert router.previous_models, "no retry breadcrumbs were recorded"
+    assert any(
+        "fallback_depth" in breadcrumb for breadcrumb in router.previous_models
+    ), "no breadcrumb carried router walk state, so this test cannot see the leak"
+    for breadcrumb in router.previous_models:
+        assert "attempted_targets" not in breadcrumb
+
+
+@pytest.mark.asyncio
+async def test_fallback_traceback_stays_available_at_debug_level():
+    """Dropping the stack from the ERROR line is only safe because the fallback path still
+    emits it once per level at DEBUG, which is what an operator needs to diagnose why every
+    fallback failed. This pins that remaining debug traceback."""
+    capture = _LogCapture(logging.DEBUG)
+
+    await _drive_cyclic_fallback(_cyclic_fallback_router(), capture)
+
+    assert any("Traceback (most recent call last)" in message for message in capture.messages)
+
+
+@pytest.mark.asyncio
+async def test_fallback_failure_detail_from_upstream_is_bounded():
+    """The detail each level records about the level below it is attacker-influenced, since
+    it carries whatever the upstream error said. It has to be bounded on its own, so a walk
+    over several groups cannot compound one large message into the log or into the message
+    handed back to the caller."""
+    huge_message = "z" * 50_000
+    capture = _LogCapture(logging.ERROR)
+
+    await _drive_cyclic_fallback(
+        _cyclic_fallback_router(),
+        capture,
+        mock_response=litellm.InternalServerError(
+            message=huge_message, llm_provider="openai", model="group-a"
+        ),
+    )
+
+    assert capture.messages, "the fallback failure path did not log at ERROR"
+    assert huge_message not in "".join(capture.messages)
+    assert max(len(message) for message in capture.messages) < 5_000