scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

test(integration): expect a pre-token invalid prompt on a bridged stream to fall back (#47020)

Browse source 
#46493 reads the first stream chunk inside the router's retry and fallback
chain, so an invalid prompt rejected before any token now falls back to the
configured fallback deployment instead of answering 400. The author
confirmed this is intended

The fallback test now asserts the spare serves the stream. The old 400
without a fallback attempt still holds when fallback_on_unsupported_params
is false, so that contract moves to a second proxy with that setting
This commit is contained in:
yuneng-jiang 2026-10-16 18:26:56 -07:00 • committed by GitHub
parent 6d0d0bb68b
commit 856a3f62f3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 52 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
tests/integration/streaming/test_responses_bridge_stream_errors.py
View file

@ -109,7 +109,9 @@ class FallbackProxy:
spare_port: int
def _fallback_config(directory: Path, primary_port: int, spare_port: int) -> Path:
def _fallback_config(
directory: Path, primary_port: int, spare_port: int, extra_router_settings: Mapping[str, JsonValue]
) -> Path:
base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
deployments: Final = [
{
@ -122,23 +124,37 @@ def _fallback_config(directory: Path, primary_port: int, spare_port: int) -> Pat
}
for name, port in ((_PRIMARY, primary_port), (_SPARE, spare_port))
]
router_settings: Final = {"num_retries": 0, "disable_cooldowns": True, "fallbacks": [{_PRIMARY: [_SPARE]}]}
router_settings: Final = {
"num_retries": 0,
"disable_cooldowns": True,
"fallbacks": [{_PRIMARY: [_SPARE]}],
**extra_router_settings,
}
path: Final = directory / "bridged-fallbacks.yaml"
path.write_text(yaml.safe_dump({**base, "model_list": deployments, "router_settings": router_settings}))
return path
@pytest.fixture(scope="module")
def fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]:
def _fallback_proxy(
tmp_path_factory: pytest.TempPathFactory, extra_router_settings: Mapping[str, JsonValue]
) -> Iterator[FallbackProxy]:
directory: Final = tmp_path_factory.mktemp("bridged-fallbacks")
primary_port, spare_port = allocate_port(), allocate_port()
with (
gateway_from_environment() as shared,
owned_proxy(shared, directory, {}, config=_fallback_config(directory, primary_port, spare_port)) as owned,
):
config: Final = _fallback_config(directory, primary_port, spare_port, extra_router_settings)
with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as owned:
yield FallbackProxy(owned, primary_port, spare_port)
@pytest.fixture(scope="module")
def fallback_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]:
yield from _fallback_proxy(tmp_path_factory, {})
@pytest.fixture(scope="module")
def bad_request_opt_out_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FallbackProxy]:
yield from _fallback_proxy(tmp_path_factory, {"fallback_on_unsupported_params": False})
def test_bridged_in_stream_rate_limit_falls_back_to_the_healthy_deployment(fallback_proxy: FallbackProxy) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with (
@ -189,9 +205,7 @@ def test_bridged_top_level_rate_limit_falls_back_to_the_healthy_deployment(fallb
_INVALID_PROMPT: Final = "Invalid prompt: your prompt was flagged."
@pytest.mark.parametrize(
_INVALID_PROMPT_REJECTIONS: Final = pytest.mark.parametrize(
"rejection",
[
pytest.param(
@ -201,7 +215,10 @@ _INVALID_PROMPT: Final = "Invalid prompt: your prompt was flagged."
pytest.param(openai_error_event("invalid_prompt", _INVALID_PROMPT), id="top_level"),
],
)
def test_bridged_invalid_prompt_answers_a_bad_request_without_trying_the_fallback(
@_INVALID_PROMPT_REJECTIONS
def test_bridged_invalid_prompt_before_any_token_falls_back_to_the_healthy_deployment(
fallback_proxy: FallbackProxy, rejection: Mapping[str, JsonValue]
) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
@ -214,6 +231,29 @@ def test_bridged_invalid_prompt_answers_a_bad_request_without_trying_the_fallbac
) as spare,
):
response: Final = _chat(fallback_proxy.gateway, chat_body(_PRIMARY, identity))
assert response.status_code == 200, response.text
assert chat_content(data_frames(response.text)) == "fallback answer", response.text
assert len(primary.drain()) == 1 and len(spare.drain()) == 1
row: Final = spend_row(response.headers["x-litellm-call-id"])
assert row["status"] == "success" and row["model_group"] == _SPARE, row
@_INVALID_PROMPT_REJECTIONS
def test_bridged_invalid_prompt_answers_a_bad_request_without_trying_the_fallback_when_bad_requests_opt_out(
bad_request_opt_out_proxy: FallbackProxy, rejection: Mapping[str, JsonValue]
) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
with (
wire_server(
serve((frame(created(identity)), frame(rejection)), AZURE_TARGET),
port=bad_request_opt_out_proxy.primary_port,
) as primary,
wire_server(
serve(healthy_stream(identity, "fallback answer"), AZURE_TARGET),
port=bad_request_opt_out_proxy.spare_port,
) as spare,
):
response: Final = _chat(bad_request_opt_out_proxy.gateway, chat_body(_PRIMARY, identity))
assert response.status_code == 400, response.text
message: Final = string_value(error_body(response)["message"])
assert message.startswith("litellm.BadRequestError: ") and _INVALID_PROMPT in message, message