From b6159e2a156d7a617d902dc4e71102423ebb27c6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:16:48 -0700 Subject: [PATCH] fix(batches): keep batch list items as OpenAI's Batch so an odd item never fails a list --- litellm/llms/openai/openai.py | 17 ++++++++++--- litellm/types/utils.py | 4 +-- tests/unit/batches/test_main.py | 44 ++++++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index ade1bae88a3..dc5e23d1d25 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.logging_utils import speech_request_body, track_ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator, MockResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.utils import ( + OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS, EmbeddingResponse, ImageResponse, LiteLLMBatch, @@ -2161,8 +2162,16 @@ class OpenAIFilesAPI(BaseLLM): return response -def _as_litellm_batches(batches: Sequence[Batch]) -> Sequence[LiteLLMBatch]: - return [LiteLLMBatch.model_validate(batch.model_dump()) for batch in batches] +def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]: + return [ + batch.model_copy( + update={ + "object": batch.object or "batch", + "status": OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(batch.status, batch.status), + } + ) + for batch in batches + ] class OpenAIBatchesAPI(BaseLLM): @@ -2355,7 +2364,7 @@ class OpenAIBatchesAPI(BaseLLM): ): verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit) response: Final = await openai_client.batches.list(after=after, limit=limit) - return response.model_copy(update={"data": _as_litellm_batches(response.data)}) + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) def list_batches( self, @@ -2394,7 +2403,7 @@ class OpenAIBatchesAPI(BaseLLM): if not isinstance(openai_client, OpenAI): raise TypeError("OpenAI client is not an instance of OpenAI. Make sure you passed a sync OpenAI client.") response: Final = openai_client.batches.list(after=after, limit=limit) - return response.model_copy(update={"data": _as_litellm_batches(response.data)}) + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) class OpenAIAssistantsAPI(BaseLLM): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3da36dd694f..fa0cf03cf94 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4785,7 +4785,7 @@ class LiteLLMFineTuningJob(FineTuningJob): _BatchFieldValue: TypeAlias = object -_OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType( +OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType( {"pending": "validating", "running": "in_progress"} ) @@ -4803,7 +4803,7 @@ class LiteLLMBatch(Batch): @field_validator("status", mode="before") @classmethod def _map_vllm_router_status(cls, value: _BatchFieldValue) -> _BatchFieldValue: - return _OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value + return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value @property def hidden_params(self) -> dict[str, _BatchFieldValue]: # mutable-ok: API requires mutation diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index bc4c365e979..a322163c5ac 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -37,13 +37,14 @@ import litellm.batches.main as bm import asyncio import datetime import json -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import Final import httpx import respx from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.integrations.custom_logger import CustomLogger +from openai.types.batch import Batch from litellm.types.utils import LiteLLMBatch @@ -1296,20 +1297,20 @@ async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected( assert [error["loc"] for error in exc_info.value.errors()] == [(field,)] -async def _list_on_vllm_router() -> list[LiteLLMBatch]: +async def _list_on_vllm_router() -> Sequence[Batch]: page: Final = await litellm.alist_batches( custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE ) return page.data -async def _list_on_vllm_router_sync() -> list[LiteLLMBatch]: +async def _list_on_vllm_router_sync() -> Sequence[Batch]: return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data @pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync)) async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches( - respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[list[LiteLLMBatch]]] + respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]] ): respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( 200, @@ -1331,3 +1332,38 @@ async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches( ("batch_5f8dac", "batch", "validating"), ("batch_c74802", "batch", "in_progress"), ] + + +async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter): + respx_mock.get("http://openai.test/v1/batches").respond( + 200, + json={ + "object": "list", + "data": [ + { + **_OPENAI_BATCH_JSON, + "status": "completed", + "usage": { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 4}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 2}, + "total_tokens": 15, + }, + }, + {**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"}, + ], + "first_id": "batch_abc123", + "last_id": "batch_def456", + "has_more": False, + }, + ) + + page: Final = await litellm.alist_batches( + custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1" + ) + + first, second = page.data + assert first.usage is not None + assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2) + assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued")