From 0b14897a7e1a3cb971606d16576169900d03a6e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:31:03 -0700 Subject: [PATCH 1/3] fix(router): pin batch, file, and fine-tuning job ids to their owning model group on fallback --- .../router_utils/fallback_event_handlers.py | 14 ++--- .../test_fallback_event_handlers.py | 40 ++++++++++++++ tests/test_litellm/test_router.py | 53 +++++++++++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 3184c1bb0c7..23eeb495d61 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -252,7 +252,7 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li return fallback_model_group, generic_fallback_idx -PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file") +PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file", "batch_id", "file_id", "fine_tuning_job_id") PROVIDER_SCOPED_CREATION_FUNCTION_NAMES: Final = frozenset({"_acreate_file"}) @@ -284,12 +284,12 @@ async def _is_fallback_target_authorized( def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: """ - True when the request names a file that only exists under one provider's credentials. + True when the request names a file, batch, or fine-tuning job that only exists under + one provider's credentials. - Batch and fine-tuning jobs are created from a file the caller already uploaded, and - that file lives in the account of the deployment that stored it. Handing the id to a - different model group can only fail, and the second provider's error replaces the - error the caller actually needs to see. + Each of those ids lives in the account of the deployment that issued it. Handing it to + a different model group asks a provider about an id it never issued, which costs an + extra round trip that can only answer not-found. """ return any(kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS) @@ -371,7 +371,7 @@ async def run_async_fallback( continue if same_model_group_only and _get_fallback_target_model_group(mg) != original_model_group: verbose_router_logger.info( - "Skipping fallback to model_group = %s: request is pinned to model_group = %s by its uploaded file", + "Skipping fallback to model_group = %s: request names a resource owned by model_group = %s", mask_sensitive_structure(mg), original_model_group, ) diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 94922e1a076..c993739e364 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -299,6 +299,46 @@ async def test_run_async_fallback_still_crosses_model_groups_without_an_uploaded assert router.attempted_model_groups == ["azure-group"] +@pytest.mark.asyncio +@pytest.mark.parametrize("resource_key", ["batch_id", "file_id", "fine_tuning_job_id"]) +async def test_run_async_fallback_keeps_provider_scoped_ids_in_their_model_group(resource_key: str): + """A batch, file, or fine-tuning job id only exists under the credentials of the group + that issued it, so a cross-group fallback asks a provider about an id it never saw.""" + router = AttemptRecordingRouter() + + with pytest.raises(RuntimeError, match="openai connection error"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["azure-group"], + original_model_group="openai-group", + original_exception=RuntimeError("openai connection error"), + max_fallbacks=3, + fallback_depth=0, + model="openai-group", + **{resource_key: "owned-by-openai"}, + ) + + assert router.attempted_model_groups == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_allows_same_model_group_retry_for_batch_cancel(): + router = AttemptRecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=[{"model": "openai-group", "_target_order": 2}], + original_model_group="openai-group", + original_exception=RuntimeError("first deployment failed"), + max_fallbacks=3, + fallback_depth=0, + model="openai-group", + batch_id="owned-by-openai", + ) + + assert router.attempted_model_groups == ["openai-group"] + + @pytest.mark.asyncio async def test_run_async_fallback_handles_explicitly_none_metadata(): """/v1/batches always sets `metadata`, and sets it to None when the caller sent diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index ff8e568a935..97286017ffe 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -558,6 +558,59 @@ async def test_async_router_acreate_file_does_not_fall_back_across_model_groups( assert "gpt-4o-mini" not in called_models +@pytest.mark.asyncio +async def test_async_router_acancel_batch_does_not_fall_back_across_model_groups(monkeypatch: pytest.MonkeyPatch): + """The proxy cancels a managed batch by handing the router the deployment id decoded + from the unified batch id. A default (``*``) fallback matches that id like any other + model string, and the fallback provider is then asked to cancel a batch it never + issued, which can only answer not-found. The router re-raises the owner's error after + that wasted round trip, so the pin's observable is the foreign call never happening.""" + import respx + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + { + "model_name": "azure-gpt", + "litellm_params": { + "model": "azure/my-azure-deployment", + "api_base": "http://127.0.0.1:9", + "api_key": "dummy-key", + "api_version": "2024-06-01", + }, + "model_info": {"id": "azure-batch-dep"}, + }, + { + "model_name": "openai-gpt", + "litellm_params": {"model": "gpt-4o-mini", "api_key": "dummy-key"}, + }, + ], + default_fallbacks=["openai-gpt"], + ) + + with respx.mock(assert_all_called=False) as respx_mock: + azure_route = respx_mock.post(host="127.0.0.1").mock( + return_value=httpx.Response(401, json={"error": {"code": "401", "message": "invalid subscription key"}}) + ) + openai_route = respx_mock.post("https://api.openai.com/v1/batches/batch_owned_by_azure/cancel").mock( + return_value=httpx.Response( + 404, + json={ + "error": { + "message": "No batch found with id 'batch_owned_by_azure'.", + "type": "invalid_request_error", + "code": "batch_not_found", + } + }, + ) + ) + with pytest.raises(openai.AuthenticationError, match="invalid subscription key"): + await router.acancel_batch(model="azure-batch-dep", batch_id="batch_owned_by_azure") + + assert azure_route.called + assert not openai_route.called + + @pytest.mark.asyncio async def test_async_router_acreate_file_uses_deployment_custom_llm_provider(): """ From 2968c246b923b949d438f886dfdc876c0653650b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 01:32:09 -0700 Subject: [PATCH 2/3] test(responses): repair two tests broken by the gpt-5 temperature gate PR #38593 stopped forwarding temperature to reasoning models, which left test_extra_body_merges_with_request_data raising UnsupportedParamsError and test_bad_request_bad_param_error no longer getting a rejection from OpenAI because drop_params now eats the param. Both repairs are the same hunks PR #38739 carries, so the branches merge clean in either order --- tests/llm_responses_api_testing/test_openai_responses_api.py | 1 + tests/openai_endpoints_tests/test_e2e_openai_responses_api.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 5f77d5a5477..cbe8204bfb5 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1818,6 +1818,7 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data input="Test", temperature=0.7, max_output_tokens=20, + reasoning={"effort": "none"}, extra_body={ "custom_field": "custom_value", }, diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index be565972b94..6abc6813761 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -115,9 +115,9 @@ def test_bad_request_error(): def test_bad_request_bad_param_error(): client = get_test_client() with pytest.raises(BadRequestError): - # Trigger error with invalid model name + # needs a non-reasoning model so drop_params keeps temperature for OpenAI to reject client.responses.create( - model="gpt-5.5", input="This should fail", temperature=2000 + model="gpt-4.1-mini", input="This should fail", temperature=2000 ) From 8cf090b3683c4a5fe31ebc9ef6f742befd4398a9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 01:46:33 -0700 Subject: [PATCH 3/3] fix(router): arm the provider-scoped fallback pin only on resource-operating handlers --- .../router_utils/fallback_event_handlers.py | 27 ++++++- .../test_fallback_event_handlers.py | 80 ++++++++++++++++++- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 23eeb495d61..924574537f3 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -253,6 +253,17 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file", "batch_id", "file_id", "fine_tuning_job_id") +PROVIDER_SCOPED_RESOURCE_FUNCTION_NAMES: Final = frozenset( + { + "_acreate_batch", + "_acancel_batch", + "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "aretrieve_fine_tuning_job", + "afile_content", + "afile_delete", + } +) PROVIDER_SCOPED_CREATION_FUNCTION_NAMES: Final = frozenset({"_acreate_file"}) @@ -284,13 +295,23 @@ async def _is_fallback_target_authorized( def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: """ - True when the request names a file, batch, or fine-tuning job that only exists under - one provider's credentials. + True when a file, batch, or fine-tuning job operation names an id that only exists + under one provider's credentials. Each of those ids lives in the account of the deployment that issued it. Handing it to a different model group asks a provider about an id it never issued, which costs an - extra round trip that can only answer not-found. + extra round trip that can only answer not-found. Generic calls dispatched through + `Router._ageneric_api_call_with_fallbacks` carry the real handler in + `original_generic_function`, so both slots are checked. Gating on the handler name + keeps completion-style requests eligible for cross-group fallback even when a caller + passes a stray extra body field that happens to share one of these key names. """ + handler_names: Final = tuple( + getattr(kwargs.get(function_key), "__name__", None) + for function_key in ("original_function", "original_generic_function") + ) + if all(name not in PROVIDER_SCOPED_RESOURCE_FUNCTION_NAMES for name in handler_names): + return False return any(kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS) diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index c993739e364..8336926c050 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -180,6 +180,30 @@ async def _acreate_file(*args: object, **kwargs: object) -> NoReturn: raise AssertionError("only used for its __name__") +async def _acancel_batch(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + +async def _acompletion(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + +async def _ageneric_api_call_with_fallbacks_helper(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + +async def acreate_fine_tuning_job(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + +async def aretrieve_fine_tuning_job(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + +async def afile_content(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + @pytest.mark.asyncio async def test_run_async_fallback_keeps_uploaded_file_requests_in_their_model_group(): """An input_file_id only exists under the credentials of the group it was uploaded @@ -217,6 +241,8 @@ async def test_run_async_fallback_keeps_fine_tuning_requests_in_their_model_grou fallback_depth=0, model="openai-group", training_file="file-owned-by-openai", + original_function=_ageneric_api_call_with_fallbacks_helper, + original_generic_function=acreate_fine_tuning_job, ) assert router.attempted_model_groups == [] @@ -300,10 +326,33 @@ async def test_run_async_fallback_still_crosses_model_groups_without_an_uploaded @pytest.mark.asyncio -@pytest.mark.parametrize("resource_key", ["batch_id", "file_id", "fine_tuning_job_id"]) -async def test_run_async_fallback_keeps_provider_scoped_ids_in_their_model_group(resource_key: str): +@pytest.mark.parametrize( + ("resource_key", "handler_kwargs"), + [ + ("batch_id", {"original_function": _acancel_batch}), + ( + "file_id", + { + "original_function": _ageneric_api_call_with_fallbacks_helper, + "original_generic_function": afile_content, + }, + ), + ( + "fine_tuning_job_id", + { + "original_function": _ageneric_api_call_with_fallbacks_helper, + "original_generic_function": aretrieve_fine_tuning_job, + }, + ), + ], +) +async def test_run_async_fallback_keeps_provider_scoped_ids_in_their_model_group( + resource_key: str, handler_kwargs: dict +): """A batch, file, or fine-tuning job id only exists under the credentials of the group - that issued it, so a cross-group fallback asks a provider about an id it never saw.""" + that issued it, so a cross-group fallback asks a provider about an id it never saw. + Generic API calls carry the real handler in original_generic_function, so the pin + must recognize it there too.""" router = AttemptRecordingRouter() with pytest.raises(RuntimeError, match="openai connection error"): @@ -316,11 +365,35 @@ async def test_run_async_fallback_keeps_provider_scoped_ids_in_their_model_group fallback_depth=0, model="openai-group", **{resource_key: "owned-by-openai"}, + **handler_kwargs, ) assert router.attempted_model_groups == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("resource_key", ["batch_id", "file_id", "fine_tuning_job_id"]) +async def test_run_async_fallback_ignores_stray_resource_ids_on_completion_calls(resource_key: str): + """A caller-supplied top-level field like file_id on a chat completion is application + data, never a provider resource reference, so it must not cost the request its + cross-group fallbacks.""" + router = AttemptRecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["azure-group"], + original_model_group="openai-group", + original_exception=RuntimeError("openai connection error"), + max_fallbacks=3, + fallback_depth=0, + model="openai-group", + original_function=_acompletion, + **{resource_key: "caller-app-data"}, + ) + + assert router.attempted_model_groups == ["azure-group"] + + @pytest.mark.asyncio async def test_run_async_fallback_allows_same_model_group_retry_for_batch_cancel(): router = AttemptRecordingRouter() @@ -334,6 +407,7 @@ async def test_run_async_fallback_allows_same_model_group_retry_for_batch_cancel fallback_depth=0, model="openai-group", batch_id="owned-by-openai", + original_function=_acancel_batch, ) assert router.attempted_model_groups == ["openai-group"]