fix(router): keep chat and other non-batch calls off Azure batch-only deployments in mixed groups

This commit is contained in:
mateo-berri 2026-10-29 04:33:40 -07:00
parent dff474e450
commit 66d6658bd6
3 changed files with 260 additions and 43 deletions

View file

@ -20,6 +20,17 @@ _ENDPOINT_PATHS: Final = TypeAdapter(tuple[str, ...])
BATCH_ENDPOINT: Final = "/v1/batch"
_AZURE_PROVIDERS: Final = frozenset({LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value})
BATCH_API_CALL_TYPES: Final = frozenset(
{
CallTypes.acreate_file.value,
CallTypes.aretrieve_batch.value,
CallTypes.acancel_batch.value,
CallTypes.afile_content.value,
CallTypes.afile_delete.value,
CallTypes.afile_list.value,
}
)
_batch_create_routing: Final[ContextVar[bool]] = ContextVar("batch_create_routing", default=False)
_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object])
@ -110,6 +121,19 @@ def _cannot_run_batches(deployment: Mapping[str, object], provider: str | None)
return provider in _AZURE_PROVIDERS and not is_batch_capable(deployment)
def _serves_only_batches(deployment: Mapping[str, object]) -> bool:
return is_batch_capable(deployment) and _deployment_provider(deployment) in _AZURE_PROVIDERS
def without_batch_only_deployments(
deployments: Sequence[_DeploymentT], call_type: str | None
) -> Sequence[_DeploymentT]:
if is_batch_create_routing() or call_type in BATCH_API_CALL_TYPES:
return deployments
serving: Final = [deployment for deployment in deployments if not _serves_only_batches(deployment)]
return serving or deployments
def without_deployments_that_cannot_batch(
deployments: Sequence[_DeploymentT], caller_deployments: Callable[[], Iterable[Mapping[str, object]]]
) -> Sequence[_DeploymentT]:

View file

@ -115,6 +115,7 @@ from litellm.litellm_core_utils.batch_utils import (
is_batch_retrieve_call_type,
replace_model_in_jsonl,
should_replace_model_in_jsonl,
without_batch_only_deployments,
without_deployments_that_cannot_batch,
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
@ -458,6 +459,7 @@ from litellm.types.services import ServiceTypes
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
PROMPT_QUOTING_ROUTING_DECISION_FIELDS,
CallTypes,
CustomPricingLiteLLMParams,
GenericBudgetConfigType,
LiteLLMBatch,
@ -7423,13 +7425,14 @@ class Router:
verbose_router_logger.debug("Inside _atext_completion()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
healthy_deployments: Final = await self.async_get_healthy_deployments(
model=model,
messages=[{"role": "user", "content": "files-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
parent_otel_span=parent_otel_span,
)
with self._route_capability_pin(CallTypes.acreate_file.value):
healthy_deployments: Final = await self.async_get_healthy_deployments(
model=model,
messages=[{"role": "user", "content": "files-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
parent_otel_span=parent_otel_span,
)
async def create_file_for_deployment(deployment: dict) -> OpenAIFileObject:
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
@ -7726,15 +7729,16 @@ class Router:
requested_model_group: Final = model
metadata_variable_name: Final = get_router_metadata_variable_name(function_name="aretrieve_batch")
if model is not None:
filtered_model_list: (
list[DeploymentTypedDict] | list[dict] | dict | None
) = await self.async_get_healthy_deployments(
model=model,
messages=[{"role": "user", "content": "retrieve-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
parent_otel_span=parent_otel_span,
)
with self._route_capability_pin(CallTypes.aretrieve_batch.value):
filtered_model_list: (
list[DeploymentTypedDict] | list[dict] | dict | None
) = await self.async_get_healthy_deployments(
model=model,
messages=[{"role": "user", "content": "retrieve-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
parent_otel_span=parent_otel_span,
)
else:
filtered_model_list = self.get_model_list()
if filtered_model_list is None:
@ -7860,12 +7864,13 @@ class Router:
try:
verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
deployment: Final = await self.async_get_available_deployment(
model=model,
messages=[{"role": "user", "content": "batch-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
with self._route_capability_pin(CallTypes.acancel_batch.value):
deployment: Final = await self.async_get_available_deployment(
model=model,
messages=[{"role": "user", "content": "batch-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id")
data: Final = deployment["litellm_params"].copy()
@ -15202,6 +15207,16 @@ class Router:
model,
)
)
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: batch-only filter narrows
healthy_deployments
if isinstance(healthy_deployments, dict)
else list(
without_batch_only_deployments(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
_routed_call_type.get(),
)
)
)
if isinstance(healthy_deployments, dict):
if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
@ -16472,6 +16487,16 @@ class Router:
model,
)
)
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: batch-only filter narrows
healthy_deployments
if isinstance(healthy_deployments, dict)
else list(
without_batch_only_deployments(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
_routed_call_type.get(),
)
)
)
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs)
pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs)

View file

@ -13,6 +13,7 @@ import threading
import warnings
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping
from datetime import datetime, timedelta, timezone
from io import BytesIO
from pathlib import Path
from types import CodeType, FrameType, MappingProxyType, SimpleNamespace
from typing import TYPE_CHECKING, Final, Literal, TypeVar, cast
@ -14014,20 +14015,40 @@ async def test_acreate_batch_on_a_mixed_wildcard_group_only_reaches_batch_deploy
assert batch_route.call_count == 20
async def _chat_deployment_ids(router: Router, model_group: str, attempts: int) -> set[str]:
return {
(
await router.acompletion(
model=model_group, messages=[{"role": "user", "content": "hi"}], mock_response="ok"
)
)._hidden_params["model_id"]
for _ in range(attempts)
}
def _mixed_azure_group() -> list[dict[str, object]]:
return [
_azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"),
_azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"),
]
@pytest.mark.asyncio
async def test_chat_calls_after_a_batch_create_still_use_every_deployment_in_the_group(
monkeypatch: pytest.MonkeyPatch,
):
"""Only the batch create itself is narrowed to mode batch deployments. Chat calls the same task makes
afterwards keep load balancing across the whole group."""
@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "latency-based-routing", "usage-based-routing"])
async def test_chat_calls_on_a_mixed_azure_group_skip_the_batch_deployment(routing_strategy: str):
"""Azure answers 400 "The current operation is not allowed in this deployment." to a chat call on a
GlobalBatch deployment, so chat calls on a group that also has a chat deployment never pick it."""
router: Final = litellm.Router(model_list=_mixed_azure_group(), routing_strategy=routing_strategy, num_retries=0)
assert await _chat_deployment_ids(router, "gpt-mixed", attempts=40) == {"chat-deployment"}
@pytest.mark.asyncio
async def test_chat_calls_after_a_batch_create_still_reach_the_chat_deployment(monkeypatch: pytest.MonkeyPatch):
"""The batch create's narrowing to mode batch deployments must not leak into the chat calls the same task
makes afterwards."""
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = litellm.Router(
model_list=[
_azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"),
_azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"),
],
num_retries=0,
)
router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0)
with respx.mock(assert_all_called=False) as respx_mock:
respx_mock.post(host="batch-sku.openai.azure.com").mock(
@ -14035,16 +14056,163 @@ async def test_chat_calls_after_a_batch_create_still_use_every_deployment_in_the
)
await _create_batches(router, "gpt-mixed", attempts=2)
chat_deployments: Final = {
(
await router.acompletion(
model="gpt-mixed", messages=[{"role": "user", "content": "hi"}], mock_response="ok"
)
)._hidden_params["model_id"]
for _ in range(40)
}
assert await _chat_deployment_ids(router, "gpt-mixed", attempts=20) == {"chat-deployment"}
assert chat_deployments == {"chat-deployment", "batch-deployment"}
@pytest.mark.asyncio
async def test_chat_calls_on_a_group_of_only_azure_batch_deployments_still_reach_them():
"""With no deployment that serves chat, a chat call keeps reaching the batch deployment, so the caller sees
Azure's own error instead of LiteLLM reporting the group as empty."""
router: Final = litellm.Router(
model_list=[
_azure_sku_deployment("gpt-batch-only", "first-batch-sku", "batch", id="first-batch"),
_azure_sku_deployment("gpt-batch-only", "second-batch-sku", "batch", id="second-batch"),
],
num_retries=0,
)
assert await _chat_deployment_ids(router, "gpt-batch-only", attempts=40) == {"first-batch", "second-batch"}
@pytest.mark.asyncio
async def test_chat_calls_keep_using_a_non_azure_batch_deployment():
"""OpenAI, Bedrock and Vertex batch through a deployment that also serves chat, so only Azure's batch-only
deployments drop out of chat routing."""
router: Final = litellm.Router(
model_list=[
{
"model_name": "gpt-openai-mixed",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "dummy-key"},
"model_info": {"id": "openai-chat", "mode": "chat"},
},
{
"model_name": "gpt-openai-mixed",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "dummy-key"},
"model_info": {"id": "openai-batch", "mode": "batch"},
},
],
num_retries=0,
)
assert await _chat_deployment_ids(router, "gpt-openai-mixed", attempts=40) == {"openai-chat", "openai-batch"}
@pytest.mark.asyncio
async def test_chat_calls_do_not_move_to_the_batch_deployment_while_the_chat_deployment_cools_down():
"""A rate limited chat deployment cools down, and the chat calls that follow answer the router's cooldown
error instead of moving to a batch deployment that would answer every one of them with a 400."""
router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0, allowed_fails=0)
with pytest.raises(litellm.RateLimitError):
await router.acompletion(
model="gpt-mixed",
messages=[{"role": "user", "content": "hi"}],
mock_response=litellm.RateLimitError(message="rate limited", llm_provider="azure", model="chat-sku"),
)
with pytest.raises(RouterRateLimitError):
await router.acompletion(model="gpt-mixed", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
_AZURE_FILE: Final = {
"id": "file-batch-input",
"object": "file",
"bytes": 2,
"created_at": 0,
"filename": "input.jsonl",
"purpose": "batch",
"status": "processed",
}
@pytest.mark.asyncio
async def test_batch_file_uploads_on_a_mixed_azure_group_still_reach_the_batch_deployment(
monkeypatch: pytest.MonkeyPatch,
):
"""A batch input file must exist on the batch deployment that later runs the batch, so file uploads keep
reaching every deployment of a mixed group."""
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0)
with respx.mock(assert_all_called=False) as respx_mock:
routes: Final = {
host: respx_mock.post(host=f"{host}.openai.azure.com").mock(
return_value=httpx.Response(200, json=_AZURE_FILE)
)
for host in ("chat-sku", "batch-sku")
}
await router.acreate_file(model="gpt-mixed", purpose="batch", file=BytesIO(b"{}"))
assert {host: route.call_count for host, route in routes.items()} == {"chat-sku": 1, "batch-sku": 1}
@pytest.mark.asyncio
async def test_batch_retrieves_on_a_mixed_azure_group_still_find_the_batch_on_the_batch_deployment(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0)
with respx.mock(assert_all_called=False) as respx_mock:
respx_mock.get(host="chat-sku.openai.azure.com").mock(
return_value=httpx.Response(404, json={"error": {"code": "NotFound", "message": "batch not found"}})
)
respx_mock.get(host="batch-sku.openai.azure.com").mock(
return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED)
)
batch: Final = await router.aretrieve_batch(model="gpt-mixed", batch_id="batch_created")
assert batch.id == "batch_created"
_BATCH_API_CALLS_ON_ONE_DEPLOYMENT: Final = {
"acancel_batch": (
"POST",
httpx.Response(200, json={**_AZURE_BATCH_CREATED, "status": "cancelling"}),
lambda router: router.acancel_batch(model="gpt-mixed", batch_id="batch_created"),
),
"afile_content": (
"GET",
httpx.Response(200, content=b"{}"),
lambda router: router.afile_content(model="gpt-mixed", file_id="file-batch-output"),
),
"afile_list": (
"GET",
httpx.Response(200, json={"object": "list", "data": [_AZURE_FILE], "has_more": False}),
lambda router: router.afile_list(model="gpt-mixed", purpose="batch"),
),
"afile_delete": (
"DELETE",
httpx.Response(200, json={"id": "file-batch-output", "object": "file", "deleted": True}),
lambda router: router.afile_delete(model="gpt-mixed", file_id="file-batch-output"),
),
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("method", "response", "call"), _BATCH_API_CALLS_ON_ONE_DEPLOYMENT.values(), ids=_BATCH_API_CALLS_ON_ONE_DEPLOYMENT
)
async def test_batch_api_calls_on_a_mixed_azure_group_can_still_reach_the_batch_deployment(
monkeypatch: pytest.MonkeyPatch,
method: str,
response: httpx.Response,
call: Callable[[Router], Awaitable[object]],
):
"""Batch jobs and their output files live on the batch deployment, so the Files and Batches calls that pick
one deployment of a mixed group keep it among their candidates."""
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = litellm.Router(model_list=_mixed_azure_group(), num_retries=0)
with respx.mock(assert_all_called=False) as respx_mock:
batch_route: Final = respx_mock.route(method=method, host="batch-sku.openai.azure.com").mock(
return_value=response
)
respx_mock.route(method=method, host="chat-sku.openai.azure.com").mock(return_value=response)
for _ in range(20):
await call(router)
assert batch_route.call_count > 0
@pytest.mark.asyncio