From 7bc2d0b06e2fd8ab1226920cc45c7b6723abd6e0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:55:41 -0700 Subject: [PATCH 1/3] fix(router): keep retry breadcrumbs per request and out of the request snapshot Retry breadcrumbs were appended to one list owned by the Router and shared by every request, and each breadcrumb copied the whole kwargs including the proxy's snapshot of the inbound request. That snapshot's body aliases the live request metadata, breadcrumbs included, so every new breadcrumb nested all the earlier ones inside itself. Memory stayed small because these are shared references, but under --detailed_debug the repr of that structure expands, so one debug line grew from 10k to 219M characters over 14 failing requests and the proxy stopped answering. Breadcrumbs now accumulate in the metadata of the request that produced them, the request snapshot is excluded from a breadcrumb, and the cap of the last 4 failed attempts applies per request. --- litellm/router.py | 58 +++++++++--------- tests/test_litellm/test_router.py | 97 ++++++++++++++++++++++++++++--- 2 files changed, 118 insertions(+), 37 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0af514fe8a2..69fe731bfbd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -586,16 +586,20 @@ set_live_deployment_replay(_replay_live_router_model_cost) # Kwargs that carry no signal about the failed attempt, so log_retry drops them from a -# breadcrumb entirely: the request payload and the router-internal walk state. Credentials are -# handled separately by mask_credentials_in_payload, which scrubs credential-named values from -# whatever kwargs remain rather than trying to enumerate every credential-bearing key here. +# breadcrumb entirely: the request payload, the proxy's snapshot of the inbound request (its body +# aliases the live request metadata, earlier breadcrumbs included, so copying it would nest every +# breadcrumb inside the next one), and the router-internal walk state. Credentials are handled +# separately by mask_credentials_in_payload, which scrubs credential-named values from whatever +# kwargs remain rather than trying to enumerate every credential-bearing key here. RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( ( "messages", "original_function", "attempted_targets", + "proxy_server_request", ) ) +RETRY_BREADCRUMB_LIMIT: Final = 4 class Router: @@ -954,7 +958,6 @@ class Router: self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model self.fail_calls: defaultdict = defaultdict(int) # dict to store fail_calls made to each model self.success_calls: defaultdict = defaultdict(int) # dict to store success_calls made to each model - self.previous_models: list = [] # list to store failed calls (passed in as metadata to next call) # make Router.chat.completions.create compatible for openai.chat.completions.create default_litellm_params = default_litellm_params or {} @@ -8048,35 +8051,30 @@ class Router: """ When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing """ - try: - _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" - # Log failed model as the previous model - previous_model: Final = { + _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var] + attempt_kwargs: Final = MappingProxyType( + {k: v for k, v in kwargs.items() if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS} + ) + attempt_metadata: Final = MappingProxyType( + {k: v for k, v in request_metadata.items() if k != "previous_models"} + ) + previous_model: Final = MappingProxyType( + { "exception_type": type(e).__name__, "exception_string": str(e), + **attempt_kwargs, + _metadata_var: attempt_metadata, } - for ( - k, - v, - ) in kwargs.items(): # log everything in kwargs except the old previous_models value - prevent nesting - if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS: - previous_model[k] = v - elif k == _metadata_var and isinstance(v, dict): - previous_model[_metadata_var] = {} - for metadata_k, metadata_v in kwargs[_metadata_var].items(): - if metadata_k != "previous_models": - previous_model[k][metadata_k] = metadata_v - - # check current size of self.previous_models, if it's larger than 3, remove the first element - if len(self.previous_models) > 3: - self.previous_models.pop(0) - - scrubbed_previous_model: Final = mask_credentials_in_payload(previous_model) - self.previous_models.append(scrubbed_previous_model) - kwargs[_metadata_var]["previous_models"] = self.previous_models - return kwargs - except Exception as e: - raise e + ) + earlier_breadcrumbs: Final = request_metadata.get("previous_models") + kept_breadcrumbs: Final[tuple[object, ...]] = ( + tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :] + if isinstance(earlier_breadcrumbs, (list, tuple)) + else () + ) + kwargs[_metadata_var]["previous_models"] = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model)) + return kwargs def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int: """ diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c843a66a1c1..2b6d4694a61 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9274,9 +9274,11 @@ class _FallbackAttemptRecorder(CustomLogger): def __init__(self): super().__init__() self.failed_targets = [] + self.breadcrumbs_per_target = [] async def log_failure_fallback_event(self, original_model_group, kwargs, original_exception): self.failed_targets.append(kwargs.get("model")) + self.breadcrumbs_per_target.append(kwargs.get("metadata", {}).get("previous_models", ())) def _cyclic_fallback_router(num_retries=0): @@ -9347,14 +9349,16 @@ async def test_retry_breadcrumbs_do_not_carry_the_walk_state(): A retry has to be configured for the walk state to reach log_retry at all.""" router = _cyclic_fallback_router(num_retries=1) capture = _LogCapture(logging.ERROR) + recorder = _FallbackAttemptRecorder() - await _drive_cyclic_fallback(router, capture) + await _drive_cyclic_fallback(router, capture, recorder) - assert router.previous_models, "no retry breadcrumbs were recorded" + breadcrumbs = [breadcrumb for hop in recorder.breadcrumbs_per_target for breadcrumb in hop] + assert breadcrumbs, "no retry breadcrumbs were recorded" assert any( - "fallback_depth" in breadcrumb for breadcrumb in router.previous_models + "fallback_depth" in breadcrumb for breadcrumb in breadcrumbs ), "no breadcrumb carried router walk state, so this test cannot see the leak" - for breadcrumb in router.previous_models: + for breadcrumb in breadcrumbs: assert "attempted_targets" not in breadcrumb @@ -9392,15 +9396,94 @@ async def test_retry_breadcrumbs_never_carry_a_forwarded_credential(container_ke container still reaches the breadcrumb, but the raw secret never does, whatever key holds it.""" router = _cyclic_fallback_router(num_retries=1) capture = _LogCapture(logging.ERROR) + metadata = {} - await _drive_cyclic_fallback(router, capture, **request_kwargs) + await _drive_cyclic_fallback(router, capture, metadata=metadata, **request_kwargs) - assert router.previous_models, "no retry breadcrumbs were recorded" - dumped = json.dumps(router.previous_models, default=str) + breadcrumbs = metadata["previous_models"] + assert breadcrumbs, "no retry breadcrumbs were recorded" + dumped = json.dumps(breadcrumbs, default=str) assert container_key in dumped, "the credential-bearing kwarg never reached the breadcrumb, so this test cannot see the leak" assert _BREADCRUMB_CREDENTIAL_CANARY not in dumped +def _always_failing_router(num_retries): + return litellm.Router( + model_list=[ + { + "model_name": "broken-group", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-fake", + "mock_response": "litellm.InternalServerError", + }, + } + ], + num_retries=num_retries, + ) + + +async def _fail_one_proxy_shaped_request(router, request_marker): + """The proxy hands the router a metadata dict and a proxy_server_request whose body is a + shallow copy of the request, so body["metadata"] is the very same dict the router later + stamps previous_models onto.""" + metadata = {"request_marker": request_marker} + with pytest.raises(litellm.InternalServerError): + await router.acompletion( + model="broken-group", + messages=[{"role": "user", "content": "hi"}], + metadata=metadata, + proxy_server_request={ + "url": "http://localhost:4000/v1/chat/completions", + "method": "POST", + "headers": {}, + "body": {"model": "broken-group", "metadata": metadata}, + }, + ) + return metadata["previous_models"] + + +def _nested_breadcrumb_lists(node): + if isinstance(node, dict): + return [v for k, v in node.items() if k == "previous_models"] + [ + found for v in node.values() for found in _nested_breadcrumb_lists(v) + ] + if isinstance(node, (list, tuple)): + return [found for item in node for found in _nested_breadcrumb_lists(item)] + return [] + + +@pytest.mark.asyncio +async def test_retry_breadcrumbs_stay_per_request_and_flat_across_failing_requests(): + """Every failed attempt appends a breadcrumb to metadata["previous_models"], and the proxy's + request snapshot aliases that same metadata dict. Kept on the Router and copied wholesale, + each breadcrumb embedded every earlier one from every earlier request, so the breadcrumb + tree, and with it the debug repr of the kwargs, roughly doubled on each failed attempt until + a single-worker proxy spent minutes in the redaction regex and stopped answering.""" + router = _always_failing_router(num_retries=2) + + breadcrumbs_per_request = [ + await _fail_one_proxy_shaped_request(router, f"request-{request_number}") for request_number in range(1, 7) + ] + + for request_number, breadcrumbs in enumerate(breadcrumbs_per_request, start=1): + assert len(breadcrumbs) == 3, "one initial attempt plus two retries failed, each leaving one breadcrumb" + assert {breadcrumb["metadata"]["request_marker"] for breadcrumb in breadcrumbs} == {f"request-{request_number}"} + for breadcrumb in breadcrumbs: + assert _nested_breadcrumb_lists(breadcrumb) == [] + assert len({len(repr(breadcrumbs)) for breadcrumbs in breadcrumbs_per_request}) == 1 + + +@pytest.mark.asyncio +async def test_retry_breadcrumbs_keep_only_the_last_four_attempts(): + router = _always_failing_router(num_retries=6) + + breadcrumbs = await _fail_one_proxy_shaped_request(router, "request-1") + + assert len(breadcrumbs) == 4 + assert [breadcrumb["metadata"]["attempted_retries"] for breadcrumb in breadcrumbs] == [3, 4, 5, 6] + + @pytest.mark.asyncio async def test_fallback_traceback_stays_available_at_debug_level(): """Dropping the stack from the ERROR line is only safe because the fallback path still From 7c87451eadffa7720536937dd2a47c714554788c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:40:25 -0700 Subject: [PATCH 2/3] chore(router): document the breadcrumb write-back and ratchet lint budgets --- basedpyright-code-budget.json | 8 ++++---- litellm/router.py | 3 ++- ruff-strict-budget.json | 8 ++++---- type-discipline-budget.json | 6 +++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 3f96531cf6f..5872a6a7257 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -3,7 +3,7 @@ "limit": 14074 }, "reportArgumentType": { - "limit": 2215 + "limit": 2214 }, "reportAssignmentType": { "limit": 319 @@ -42,7 +42,7 @@ "limit": 12 }, "reportIndexIssue": { - "limit": 25 + "limit": 24 }, "reportInvalidTypeForm": { "limit": 34 @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15290 + "limit": 15289 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,7 +105,7 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38332 + "limit": 38328 }, "reportUnknownParameterType": { "limit": 19625 diff --git a/litellm/router.py b/litellm/router.py index 69fe731bfbd..11a042861f8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8073,7 +8073,8 @@ class Router: if isinstance(earlier_breadcrumbs, (list, tuple)) else () ) - kwargs[_metadata_var]["previous_models"] = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model)) + breadcrumbs: Final = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model)) + kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict return kwargs def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 4fcf650a8bc..d7cf9e039bc 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -144,7 +144,7 @@ "limit": 1 }, "PLR1704": { - "limit": 3 + "limit": 2 }, "PLR1714": { "limit": 253 @@ -240,13 +240,13 @@ "limit": 96 }, "TRY201": { - "limit": 403 + "limit": 402 }, "TRY203": { - "limit": 111 + "limit": 110 }, "TRY300": { - "limit": 854 + "limit": 853 }, "UP028": { "limit": 2 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index f3c4c7760c6..d73b1b2e277 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22334 + "limit": 22333 }, "LIT002": { - "limit": 26763 + "limit": 26760 }, "LIT003": { "limit": 261 @@ -30,7 +30,7 @@ "limit": 16480 }, "LIT011": { - "limit": 5520 + "limit": 5518 }, "LIT012": { "limit": 4489 From 9c795e52f3a9bc012872bc5f05ff2af4758a4d4f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:55:32 -0700 Subject: [PATCH 3/3] chore(lint): drop budget limits back to the values on the merged base The staging merge resolved three budget conflicts by keeping this branch's older, higher numbers, which turned budget-ratchet red. Nothing on the branch adds violations for those rules, so the base's limits hold. --- basedpyright-code-budget.json | 2 +- type-discipline-budget.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 4cbe9661d02..60136bb0650 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -123,7 +123,7 @@ "limit": 4 }, "reportUnnecessaryIsInstance": { - "limit": 823 + "limit": 819 }, "reportUntypedBaseClass": { "limit": 0 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 4837cc9e75f..1474171717f 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22332 + "limit": 22330 }, "LIT002": { "limit": 26760 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16480 + "limit": 16478 }, "LIT011": { "limit": 5518