From 4d4cec65b8d8cc1eb696054f4abe4b94c0760288 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:34:26 -0700 Subject: [PATCH] fix: return the provider page for an async client in sync batch list and pass through non-string list statuses --- litellm/llms/openai/openai.py | 11 ++++--- litellm/types/utils.py | 6 +++- tests/unit/batches/test_main.py | 19 +++++++++++ tests/unit/llms/openai/test_openai.py | 46 ++++++++++++++++++++------- 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index dc5e23d1d25..956a8aaea01 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -16,6 +16,7 @@ from openai._base_client import make_request_options from openai._constants import RAW_RESPONSE_HEADER from openai._legacy_response import LegacyAPIResponse from openai._types import RequestOptions +from openai.pagination import SyncCursorPage from openai.types import CreateEmbeddingResponse from openai.types.beta.assistant_deleted import AssistantDeleted from openai.types.chat import ChatCompletion, ChatCompletionChunk @@ -34,12 +35,12 @@ from litellm.litellm_core_utils.logging_utils import speech_request_body, track_ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator, MockResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.utils import ( - OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS, EmbeddingResponse, ImageResponse, LiteLLMBatch, ModelResponse, ModelResponseStream, + openai_batch_status, ) from litellm.utils import ( CustomStreamWrapper, @@ -2167,7 +2168,7 @@ def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]: batch.model_copy( update={ "object": batch.object or "batch", - "status": OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(batch.status, batch.status), + "status": openai_batch_status(batch.status), } ) for batch in batches @@ -2376,7 +2377,7 @@ class OpenAIBatchesAPI(BaseLLM): organization: str | None, after: str | None = None, limit: int | None = None, - client: OpenAI | None = None, + client: OpenAI | AsyncOpenAI | None = None, litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( @@ -2400,9 +2401,9 @@ class OpenAIBatchesAPI(BaseLLM): "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." ) return self.alist_batches(openai_client=openai_client, after=after, limit=limit) - if not isinstance(openai_client, OpenAI): - raise TypeError("OpenAI client is not an instance of OpenAI. Make sure you passed a sync OpenAI client.") response: Final = openai_client.batches.list(after=after, limit=limit) + if not isinstance(response, SyncCursorPage): + return response return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d0cbe42144f..188e1384992 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4900,6 +4900,10 @@ OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType( ) +def openai_batch_status(status: _JsonValue) -> _JsonValue: + return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status + + class LiteLLMBatch(Batch): _hidden_params: dict = {} @@ -4913,7 +4917,7 @@ class LiteLLMBatch(Batch): @field_validator("status", mode="before") @classmethod def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue: - return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value + return openai_batch_status(value) @property def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index a322163c5ac..8b22395f462 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -1334,6 +1334,25 @@ async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches( ] +async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string( + respx_mock: respx.MockRouter, +): + respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( + 200, + json={ + "object": "list", + "data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) + + batches: Final = await _list_on_vllm_router() + + assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])] + + async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter): respx_mock.get("http://openai.test/v1/batches").respond( 200, diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 603463d3ce3..019163aa7b1 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -2051,16 +2051,38 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope assert [message["role"] for message in messages] == expected_roles -def test_sync_batch_list_refuses_an_async_client_instead_of_returning_an_unawaited_page(): - async_client: Final = AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1") +@pytest.mark.asyncio +async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await( + respx_mock: respx.MockRouter, +): + respx_mock.get("http://vllm-router.test/v1/batches").respond( + 200, + json={ + "object": "list", + "data": [ + { + "id": "batch_5f8dac", + "status": "pending", + "input_file_id": "file-b73f4c", + "created_at": 1793239207, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + } + ], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) - with pytest.raises(TypeError, match="not an instance of OpenAI"): - OpenAIBatchesAPI().list_batches( - _is_async=False, - api_key="sk-test", - api_base="http://vllm-router.test/v1", - timeout=5.0, - max_retries=0, - organization=None, - client=async_client, - ) + page: Final = OpenAIBatchesAPI().list_batches( + _is_async=False, + api_key="sk-test", + api_base="http://vllm-router.test/v1", + timeout=5.0, + max_retries=0, + organization=None, + client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1"), + ) + + assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]