From db8d4924b075e53acad64082d80395fdea57dea6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 04:25:40 +0000 Subject: [PATCH] test(router): run the streaming fallback metadata suite on both backends Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Aw1xktpUnix5tWhvSzey8t --- litellm-rust/crates/router/POC.md | 2 ++ tests/unit/router_backends/backends.py | 2 ++ tests/unit/test_router_streaming_fallback_metadata.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/litellm-rust/crates/router/POC.md b/litellm-rust/crates/router/POC.md index f33982ec3b1..25345226fea 100644 --- a/litellm-rust/crates/router/POC.md +++ b/litellm-rust/crates/router/POC.md @@ -179,6 +179,8 @@ Streaming (chat, both drivers): `RoutedCall` reads the stream until the first ch Responses API (`RustRouter.aresponses`): `RoutedCall._ageneric` mirrors `_ageneric_api_call_with_fallbacks_helper` after selection (metadata in `litellm_metadata`, failures counted by model group, the Anthropic refusal check). `rust_streams.responses_until_output` holds the opening lifecycle events until the first output, as Python's wrapper does; a mid-stream fallback error before then goes to the hop's chain with a content-policy trigger unwrapped. After output the stream still falls back: the `ResponsesReplay` (Python's `FallbackResponsesStreamWrapper`, moved to `litellm/router_utils/responses_fallback_stream.py` so both backends use it) calls `Engine::resume` through a second `route` with the hop's group, depth, original group and attempted targets, and the input continued from the generated text. Partial usage is added to the fallback's terminal events with Python's helpers (not covered by a parity test yet). Headers after a fallback that falls back again differ on purpose (spec section 11 item 29). Anthropic messages (`RustRouter.aanthropic_messages`): `rust_streams.anthropic_until_committed` holds lifecycle frames until the first content delta or 200 frames, and fails the attempt on a recoverable error frame or a safeguard refusal, deciding with `PythonRouter`'s own gates (`_anthropic_messages_stream_can_retry`, `_anthropic_messages_stream_can_fall_back`, `_anthropic_messages_recoverable_frame_error`, `_refusal_fallback_available`) over the attempt's request. When nothing could take over it commits at once and forwards every frame. Pings, which Python forwards live while frames are held, replay ahead of the held frames, so the caller sees the same order but only once the attempt commits. A failure before the commit goes to the engine's `retry_stream` (`_aanthropic_messages_retry_same_group`): the committed, policy or plain budget, `should_retry_this_error` unless a policy governs, backoff, then the hop's fallback chain; a retry whose stream fails again carries on the same count. Retry decisions and cooldowns read the provider error inside the envelope (`Classified.original`), the fallback chain reads the envelope with content-policy and context-window errors unwrapped. A retry failing before its stream opens comes back in Python's envelope (`stream_retry` on the attempt), with `retries_pre_stream` saying whether the loop goes on +Suite survey (2026-10-07, every test run on both backends with Rust pinned to `RUST_REQUIRED`): no Rust-only failure was a wrong routing decision. In `tests/local_testing/test_router_{fallbacks,retries,timeout}.py`, 23 tests pass on Python and fail on Rust: 11 use configs the backend declines (rpm/tpm, wildcards, routing strategies, dict fallback entries, `allowed_fails_policy`) and 12 patch a `PythonRouter` internal (`should_retry_this_error`, `_time_to_sleep_before_retry`, `make_call`, `log_retry`, `_get_stream_timeout`); those modules also hold live tests that need provider keys, so they stay Python-only. In `tests/unit/test_router/test_router.py` 777 of 992 fail on Rust: 248 declined configs, and the rest call `PythonRouter` internals (`async_function_with_fallbacks_common_utils`, the Anthropic stream helpers, ...) or read views increment 4 has to provide (`get_model_group_info`, `get_deployment_by_model_group_name`, `get_deployment_model_info`, `get_model_list`, `get_model_access_groups`, `get_deployment_credentials_with_provider`, ...). Suites that test Python helpers directly (`router_utils/test_fallback_event_handlers.py`, `test_cooldown_handlers.py`, `litellm_core_utils/test_fallback_generalizations.py`) never call the router's own methods, so running them twice adds nothing. Opted in: `tests/unit/test_router_streaming_fallback_metadata.py`. The parity suites under `tests/test_litellm_rust/router` carry the behavior coverage instead + Next, following the revised POC scope: finish increment 1 (more suites opted into `router_backend`, starting with `tests/unit/test_router_order_fallback.py` and the fallback/cooldown tests in `tests/unit/router_utils` that go through `Router(...)`), then increments 2, 3 and 4. Then sketch the options for increment 3 Known gaps to close or decline: CustomLogger fallback-event hooks and `router_cooldown_event_callback` (increment 3), the 200-key `InMemoryCache` eviction, fallback keys containing `/` (provider-prefixed matching needs the cost map) diff --git a/tests/unit/router_backends/backends.py b/tests/unit/router_backends/backends.py index 1ccb99f824a..c88eb5a8773 100644 --- a/tests/unit/router_backends/backends.py +++ b/tests/unit/router_backends/backends.py @@ -33,6 +33,8 @@ RUST_GAPS: Final[Mapping[str, str]] = MappingProxyType( "test_request_num_retries_still_wins_on_image_generation": "aimage_generation is not served yet", "test_flag_off_hides_fallback_credentials": "dict fallback targets decline at construction", "test_flag_on_masks_fallback_credentials": "dict fallback targets decline at construction", + "test_include_fallback_errors_propagates_through_router": "include_fallback_errors is not served yet", + "test_set_response_headers_adds_model_group_to_streaming_wrapper": "calls a PythonRouter internal the Rust backend replaces", } ) diff --git a/tests/unit/test_router_streaming_fallback_metadata.py b/tests/unit/test_router_streaming_fallback_metadata.py index 6ed70dc7cfe..90a2c9a9bd7 100644 --- a/tests/unit/test_router_streaming_fallback_metadata.py +++ b/tests/unit/test_router_streaming_fallback_metadata.py @@ -7,6 +7,9 @@ import litellm from litellm.proxy.proxy_server import _should_include_fallback_errors from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict +from tests.unit.router_backends.backends import router_backend # noqa: F401 # fixture import + +pytestmark = pytest.mark.usefixtures("router_backend") def test_apply_fallback_hidden_params_copies_from_fallback_response():