From 856a3f62f3eb79694d0ed22c6fa5dc7982f358ab Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 16 Oct 2026 18:26:56 -0700 Subject: [PATCH] test(integration): expect a pre-token invalid prompt on a bridged stream to fall back (#47020) #46493 reads the first stream chunk inside the router's retry and fallback chain, so an invalid prompt rejected before any token now falls back to the configured fallback deployment instead of answering 400. The author confirmed this is intended The fallback test now asserts the spare serves the stream. The old 400 without a fallback attempt still holds when fallback_on_unsupported_params is false, so that contract moves to a second proxy with that setting --- .../test_responses_bridge_stream_errors.py | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/tests/integration/streaming/test_responses_bridge_stream_errors.py b/tests/integration/streaming/test_responses_bridge_stream_errors.py index 3fba69b39b7..9e8f456df49 100644 --- a/tests/integration/streaming/test_responses_bridge_stream_errors.py +++ b/tests/integration/streaming/test_responses_bridge_stream_errors.py @@ -109,7 +109,9 @@ class FallbackProxy: spare_port: int -def _fallback_config(directory: Path, primary_port: int, spare_port: int) -> Path: +def _fallback_config( + directory: Path, primary_port: int, spare_port: int, extra_router_settings: Mapping[str, JsonValue] +) -> Path: base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) deployments: Final = [ { @@ -122,23 +124,37 @@ def _fallback_config(directory: Path, primary_port: int, spare_port: int) -> Pat } for name, port in ((_PRIMARY, primary_port), (_SPARE, spare_port)) ] - router_settings: Final = {"num_retries": 0, "disable_cooldowns": True, "fallbacks": [{_PRIMARY: [_SPARE]}]} + router_settings: Final = { + "num_retries": 0, + "disable_cooldowns": True, + "fallbacks": [{_PRIMARY: [_SPARE]}], + **extra_router_settings, + } path: Final = directory / "bridged-fallbacks.yaml" path.write_text(yaml.safe_dump({**base, "model_list": deployments, "router_settings": router_settings})) return path -@pytest.fixture(scope="module") -def fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]: +def _fallback_proxy( + tmp_path_factory: pytest.TempPathFactory, extra_router_settings: Mapping[str, JsonValue] +) -> Iterator[FallbackProxy]: directory: Final = tmp_path_factory.mktemp("bridged-fallbacks") primary_port, spare_port = allocate_port(), allocate_port() - with ( - gateway_from_environment() as shared, - owned_proxy(shared, directory, {}, config=_fallback_config(directory, primary_port, spare_port)) as owned, - ): + config: Final = _fallback_config(directory, primary_port, spare_port, extra_router_settings) + with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as owned: yield FallbackProxy(owned, primary_port, spare_port) +@pytest.fixture(scope="module") +def fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]: + yield from _fallback_proxy(tmp_path_factory, {}) + + +@pytest.fixture(scope="module") +def bad_request_opt_out_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]: + yield from _fallback_proxy(tmp_path_factory, {"fallback_on_unsupported_params": False}) + + def test_bridged_in_stream_rate_limit_falls_back_to_the_healthy_deployment(fallback_proxy: FallbackProxy) -> None: identity: Final = "resp_" + uuid.uuid4().hex with ( @@ -189,9 +205,7 @@ def test_bridged_top_level_rate_limit_falls_back_to_the_healthy_deployment(fallb _INVALID_PROMPT: Final = "Invalid prompt: your prompt was flagged." - - -@pytest.mark.parametrize( +_INVALID_PROMPT_REJECTIONS: Final = pytest.mark.parametrize( "rejection", [ pytest.param( @@ -201,7 +215,10 @@ _INVALID_PROMPT: Final = "Invalid prompt: your prompt was flagged." pytest.param(openai_error_event("invalid_prompt", _INVALID_PROMPT), id="top_level"), ], ) -def test_bridged_invalid_prompt_answers_a_bad_request_without_trying_the_fallback( + + +@_INVALID_PROMPT_REJECTIONS +def test_bridged_invalid_prompt_before_any_token_falls_back_to_the_healthy_deployment( fallback_proxy: FallbackProxy, rejection: Mapping[str, JsonValue] ) -> None: identity: Final = "resp_" + uuid.uuid4().hex @@ -214,6 +231,29 @@ def test_bridged_invalid_prompt_answers_a_bad_request_without_trying_the_fallbac ) as spare, ): response: Final = _chat(fallback_proxy.gateway, chat_body(_PRIMARY, identity)) + assert response.status_code == 200, response.text + assert chat_content(data_frames(response.text)) == "fallback answer", response.text + assert len(primary.drain()) == 1 and len(spare.drain()) == 1 + row: Final = spend_row(response.headers["x-litellm-call-id"]) + assert row["status"] == "success" and row["model_group"] == _SPARE, row + + +@_INVALID_PROMPT_REJECTIONS +def test_bridged_invalid_prompt_answers_a_bad_request_without_trying_the_fallback_when_bad_requests_opt_out( + bad_request_opt_out_proxy: FallbackProxy, rejection: Mapping[str, JsonValue] +) -> None: + identity: Final = "resp_" + uuid.uuid4().hex + with ( + wire_server( + serve((frame(created(identity)), frame(rejection)), AZURE_TARGET), + port=bad_request_opt_out_proxy.primary_port, + ) as primary, + wire_server( + serve(healthy_stream(identity, "fallback answer"), AZURE_TARGET), + port=bad_request_opt_out_proxy.spare_port, + ) as spare, + ): + response: Final = _chat(bad_request_opt_out_proxy.gateway, chat_body(_PRIMARY, identity)) assert response.status_code == 400, response.text message: Final = string_value(error_body(response)["message"]) assert message.startswith("litellm.BadRequestError: ") and _INVALID_PROMPT in message, message