From 66d6658bd69503540b2a16df981ed68f42e823f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:33:40 -0700 Subject: [PATCH] fix(router): keep chat and other non-batch calls off Azure batch-only deployments in mixed groups --- litellm/litellm_core_utils/batch_utils.py | 24 +++ litellm/router.py | 69 ++++--- tests/unit/test_router/test_router.py | 210 +++++++++++++++++++--- 3 files changed, 260 insertions(+), 43 deletions(-) diff --git a/litellm/litellm_core_utils/batch_utils.py b/litellm/litellm_core_utils/batch_utils.py index 5b94d4b4e5b..f930961f42c 100644 --- a/litellm/litellm_core_utils/batch_utils.py +++ b/litellm/litellm_core_utils/batch_utils.py @@ -20,6 +20,17 @@ _ENDPOINT_PATHS: Final = TypeAdapter(tuple[str, ...]) BATCH_ENDPOINT: Final = "/v1/batch" _AZURE_PROVIDERS: Final = frozenset({LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value}) +BATCH_API_CALL_TYPES: Final = frozenset( + { + CallTypes.acreate_file.value, + CallTypes.aretrieve_batch.value, + CallTypes.acancel_batch.value, + CallTypes.afile_content.value, + CallTypes.afile_delete.value, + CallTypes.afile_list.value, + } +) + _batch_create_routing: Final[ContextVar[bool]] = ContextVar("batch_create_routing", default=False) _DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object]) @@ -110,6 +121,19 @@ def _cannot_run_batches(deployment: Mapping[str, object], provider: str | None) return provider in _AZURE_PROVIDERS and not is_batch_capable(deployment) +def _serves_only_batches(deployment: Mapping[str, object]) -> bool: + return is_batch_capable(deployment) and _deployment_provider(deployment) in _AZURE_PROVIDERS + + +def without_batch_only_deployments( + deployments: Sequence[_DeploymentT], call_type: str | None +) -> Sequence[_DeploymentT]: + if is_batch_create_routing() or call_type in BATCH_API_CALL_TYPES: + return deployments + serving: Final = [deployment for deployment in deployments if not _serves_only_batches(deployment)] + return serving or deployments + + def without_deployments_that_cannot_batch( deployments: Sequence[_DeploymentT], caller_deployments: Callable[[], Iterable[Mapping[str, object]]] ) -> Sequence[_DeploymentT]: diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..5728c2e7bf9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -115,6 +115,7 @@ from litellm.litellm_core_utils.batch_utils import ( is_batch_retrieve_call_type, replace_model_in_jsonl, should_replace_model_in_jsonl, + without_batch_only_deployments, without_deployments_that_cannot_batch, ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout @@ -458,6 +459,7 @@ from litellm.types.services import ServiceTypes from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, PROMPT_QUOTING_ROUTING_DECISION_FIELDS, + CallTypes, CustomPricingLiteLLMParams, GenericBudgetConfigType, LiteLLMBatch, @@ -7423,13 +7425,14 @@ class Router: verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - healthy_deployments: Final = await self.async_get_healthy_deployments( - model=model, - messages=[{"role": "user", "content": "files-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - parent_otel_span=parent_otel_span, - ) + with self._route_capability_pin(CallTypes.acreate_file.value): + healthy_deployments: Final = await self.async_get_healthy_deployments( + model=model, + messages=[{"role": "user", "content": "files-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + parent_otel_span=parent_otel_span, + ) async def create_file_for_deployment(deployment: dict) -> OpenAIFileObject: from litellm.litellm_core_utils.core_helpers import safe_deep_copy @@ -7726,15 +7729,16 @@ class Router: requested_model_group: Final = model metadata_variable_name: Final = get_router_metadata_variable_name(function_name="aretrieve_batch") if model is not None: - filtered_model_list: ( - list[DeploymentTypedDict] | list[dict] | dict | None - ) = await self.async_get_healthy_deployments( - model=model, - messages=[{"role": "user", "content": "retrieve-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - parent_otel_span=parent_otel_span, - ) + with self._route_capability_pin(CallTypes.aretrieve_batch.value): + filtered_model_list: ( + list[DeploymentTypedDict] | list[dict] | dict | None + ) = await self.async_get_healthy_deployments( + model=model, + messages=[{"role": "user", "content": "retrieve-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + parent_otel_span=parent_otel_span, + ) else: filtered_model_list = self.get_model_list() if filtered_model_list is None: @@ -7860,12 +7864,13 @@ class Router: try: verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "batch-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with self._route_capability_pin(CallTypes.acancel_batch.value): + deployment: Final = await self.async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "batch-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id") data: Final = deployment["litellm_params"].copy() @@ -15202,6 +15207,16 @@ class Router: model, ) ) + healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: batch-only filter narrows + healthy_deployments + if isinstance(healthy_deployments, dict) + else list( + without_batch_only_deployments( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + _routed_call_type.get(), + ) + ) + ) if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: @@ -16472,6 +16487,16 @@ class Router: model, ) ) + healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: batch-only filter narrows + healthy_deployments + if isinstance(healthy_deployments, dict) + else list( + without_batch_only_deployments( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + _routed_call_type.get(), + ) + ) + ) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b55b3c5efc2..8c43c39fe7e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -13,6 +13,7 @@ import threading import warnings from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping from datetime import datetime, timedelta, timezone +from io import BytesIO from pathlib import Path from types import CodeType, FrameType, MappingProxyType, SimpleNamespace from typing import TYPE_CHECKING, Final, Literal, TypeVar, cast @@ -14014,20 +14015,40 @@ async def test_acreate_batch_on_a_mixed_wildcard_group_only_reaches_batch_deploy assert batch_route.call_count == 20 +async def _chat_deployment_ids(router: Router, model_group: str, attempts: int) -> set[str]: + return { + ( + await router.acompletion( + model=model_group, messages=[{"role": "user", "content": "hi"}], mock_response="ok" + ) + )._hidden_params["model_id"] + for _ in range(attempts) + } + + +def _mixed_azure_group() -> list[dict[str, object]]: + return [ + _azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"), + _azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"), + ] + + @pytest.mark.asyncio -async def test_chat_calls_after_a_batch_create_still_use_every_deployment_in_the_group( - monkeypatch: pytest.MonkeyPatch, -): - """Only the batch create itself is narrowed to mode batch deployments. Chat calls the same task makes - afterwards keep load balancing across the whole group.""" +@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "latency-based-routing", "usage-based-routing"]) +async def test_chat_calls_on_a_mixed_azure_group_skip_the_batch_deployment(routing_strategy: str): + """Azure answers 400 "The current operation is not allowed in this deployment." to a chat call on a + GlobalBatch deployment, so chat calls on a group that also has a chat deployment never pick it.""" + router: Final = litellm.Router(model_list=_mixed_azure_group(), routing_strategy=routing_strategy, num_retries=0) + + assert await _chat_deployment_ids(router, "gpt-mixed", attempts=40) == {"chat-deployment"} + + +@pytest.mark.asyncio +async def test_chat_calls_after_a_batch_create_still_reach_the_chat_deployment(monkeypatch: pytest.MonkeyPatch): + """The batch create's narrowing to mode batch deployments must not leak into the chat calls the same task + makes afterwards.""" monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - router: Final = litellm.Router( - model_list=[ - _azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"), - _azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"), - ], - num_retries=0, - ) + router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0) with respx.mock(assert_all_called=False) as respx_mock: respx_mock.post(host="batch-sku.openai.azure.com").mock( @@ -14035,16 +14056,163 @@ async def test_chat_calls_after_a_batch_create_still_use_every_deployment_in_the ) await _create_batches(router, "gpt-mixed", attempts=2) - chat_deployments: Final = { - ( - await router.acompletion( - model="gpt-mixed", messages=[{"role": "user", "content": "hi"}], mock_response="ok" - ) - )._hidden_params["model_id"] - for _ in range(40) - } + assert await _chat_deployment_ids(router, "gpt-mixed", attempts=20) == {"chat-deployment"} - assert chat_deployments == {"chat-deployment", "batch-deployment"} + +@pytest.mark.asyncio +async def test_chat_calls_on_a_group_of_only_azure_batch_deployments_still_reach_them(): + """With no deployment that serves chat, a chat call keeps reaching the batch deployment, so the caller sees + Azure's own error instead of LiteLLM reporting the group as empty.""" + router: Final = litellm.Router( + model_list=[ + _azure_sku_deployment("gpt-batch-only", "first-batch-sku", "batch", id="first-batch"), + _azure_sku_deployment("gpt-batch-only", "second-batch-sku", "batch", id="second-batch"), + ], + num_retries=0, + ) + + assert await _chat_deployment_ids(router, "gpt-batch-only", attempts=40) == {"first-batch", "second-batch"} + + +@pytest.mark.asyncio +async def test_chat_calls_keep_using_a_non_azure_batch_deployment(): + """OpenAI, Bedrock and Vertex batch through a deployment that also serves chat, so only Azure's batch-only + deployments drop out of chat routing.""" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-openai-mixed", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "dummy-key"}, + "model_info": {"id": "openai-chat", "mode": "chat"}, + }, + { + "model_name": "gpt-openai-mixed", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "dummy-key"}, + "model_info": {"id": "openai-batch", "mode": "batch"}, + }, + ], + num_retries=0, + ) + + assert await _chat_deployment_ids(router, "gpt-openai-mixed", attempts=40) == {"openai-chat", "openai-batch"} + + +@pytest.mark.asyncio +async def test_chat_calls_do_not_move_to_the_batch_deployment_while_the_chat_deployment_cools_down(): + """A rate limited chat deployment cools down, and the chat calls that follow answer the router's cooldown + error instead of moving to a batch deployment that would answer every one of them with a 400.""" + router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0, allowed_fails=0) + + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="gpt-mixed", + messages=[{"role": "user", "content": "hi"}], + mock_response=litellm.RateLimitError(message="rate limited", llm_provider="azure", model="chat-sku"), + ) + + with pytest.raises(RouterRateLimitError): + await router.acompletion(model="gpt-mixed", messages=[{"role": "user", "content": "hi"}], mock_response="ok") + + +_AZURE_FILE: Final = { + "id": "file-batch-input", + "object": "file", + "bytes": 2, + "created_at": 0, + "filename": "input.jsonl", + "purpose": "batch", + "status": "processed", +} + + +@pytest.mark.asyncio +async def test_batch_file_uploads_on_a_mixed_azure_group_still_reach_the_batch_deployment( + monkeypatch: pytest.MonkeyPatch, +): + """A batch input file must exist on the batch deployment that later runs the batch, so file uploads keep + reaching every deployment of a mixed group.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0) + + with respx.mock(assert_all_called=False) as respx_mock: + routes: Final = { + host: respx_mock.post(host=f"{host}.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_FILE) + ) + for host in ("chat-sku", "batch-sku") + } + await router.acreate_file(model="gpt-mixed", purpose="batch", file=BytesIO(b"{}")) + + assert {host: route.call_count for host, route in routes.items()} == {"chat-sku": 1, "batch-sku": 1} + + +@pytest.mark.asyncio +async def test_batch_retrieves_on_a_mixed_azure_group_still_find_the_batch_on_the_batch_deployment( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0) + + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.get(host="chat-sku.openai.azure.com").mock( + return_value=httpx.Response(404, json={"error": {"code": "NotFound", "message": "batch not found"}}) + ) + respx_mock.get(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + batch: Final = await router.aretrieve_batch(model="gpt-mixed", batch_id="batch_created") + + assert batch.id == "batch_created" + + +_BATCH_API_CALLS_ON_ONE_DEPLOYMENT: Final = { + "acancel_batch": ( + "POST", + httpx.Response(200, json={**_AZURE_BATCH_CREATED, "status": "cancelling"}), + lambda router: router.acancel_batch(model="gpt-mixed", batch_id="batch_created"), + ), + "afile_content": ( + "GET", + httpx.Response(200, content=b"{}"), + lambda router: router.afile_content(model="gpt-mixed", file_id="file-batch-output"), + ), + "afile_list": ( + "GET", + httpx.Response(200, json={"object": "list", "data": [_AZURE_FILE], "has_more": False}), + lambda router: router.afile_list(model="gpt-mixed", purpose="batch"), + ), + "afile_delete": ( + "DELETE", + httpx.Response(200, json={"id": "file-batch-output", "object": "file", "deleted": True}), + lambda router: router.afile_delete(model="gpt-mixed", file_id="file-batch-output"), + ), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("method", "response", "call"), _BATCH_API_CALLS_ON_ONE_DEPLOYMENT.values(), ids=_BATCH_API_CALLS_ON_ONE_DEPLOYMENT +) +async def test_batch_api_calls_on_a_mixed_azure_group_can_still_reach_the_batch_deployment( + monkeypatch: pytest.MonkeyPatch, + method: str, + response: httpx.Response, + call: Callable[[Router], Awaitable[object]], +): + """Batch jobs and their output files live on the batch deployment, so the Files and Batches calls that pick + one deployment of a mixed group keep it among their candidates.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0) + + with respx.mock(assert_all_called=False) as respx_mock: + batch_route: Final = respx_mock.route(method=method, host="batch-sku.openai.azure.com").mock( + return_value=response + ) + respx_mock.route(method=method, host="chat-sku.openai.azure.com").mock(return_value=response) + for _ in range(20): + await call(router) + + assert batch_route.call_count > 0 @pytest.mark.asyncio