fix(batches): keep batch list items as OpenAI's Batch so an odd item never fails a list

This commit is contained in:
mateo-berri 2026-10-29 18:16:48 -07:00
parent 1d6527c7ac
commit b6159e2a15
3 changed files with 55 additions and 10 deletions

View file

@ -34,6 +34,7 @@ from litellm.litellm_core_utils.logging_utils import speech_request_body, track_
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator, MockResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.utils import (
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS,
EmbeddingResponse,
ImageResponse,
LiteLLMBatch,
@ -2161,8 +2162,16 @@ class OpenAIFilesAPI(BaseLLM):
return response
def _as_litellm_batches(batches: Sequence[Batch]) -> Sequence[LiteLLMBatch]:
return [LiteLLMBatch.model_validate(batch.model_dump()) for batch in batches]
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
return [
batch.model_copy(
update={
"object": batch.object or "batch",
"status": OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(batch.status, batch.status),
}
)
for batch in batches
]
class OpenAIBatchesAPI(BaseLLM):
@ -2355,7 +2364,7 @@ class OpenAIBatchesAPI(BaseLLM):
):
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
response: Final = await openai_client.batches.list(after=after, limit=limit)
return response.model_copy(update={"data": _as_litellm_batches(response.data)})
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
def list_batches(
self,
@ -2394,7 +2403,7 @@ class OpenAIBatchesAPI(BaseLLM):
if not isinstance(openai_client, OpenAI):
raise TypeError("OpenAI client is not an instance of OpenAI. Make sure you passed a sync OpenAI client.")
response: Final = openai_client.batches.list(after=after, limit=limit)
return response.model_copy(update={"data": _as_litellm_batches(response.data)})
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
class OpenAIAssistantsAPI(BaseLLM):

View file

@ -4785,7 +4785,7 @@ class LiteLLMFineTuningJob(FineTuningJob):
_BatchFieldValue: TypeAlias = object
_OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
{"pending": "validating", "running": "in_progress"}
)
@ -4803,7 +4803,7 @@ class LiteLLMBatch(Batch):
@field_validator("status", mode="before")
@classmethod
def _map_vllm_router_status(cls, value: _BatchFieldValue) -> _BatchFieldValue:
return _OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value
@property
def hidden_params(self) -> dict[str, _BatchFieldValue]: # mutable-ok: API requires mutation

View file

@ -37,13 +37,14 @@ import litellm.batches.main as bm
import asyncio
import datetime
import json
from collections.abc import Awaitable, Callable, Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import Final
import httpx
import respx
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_logger import CustomLogger
from openai.types.batch import Batch
from litellm.types.utils import LiteLLMBatch
@ -1296,20 +1297,20 @@ async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected(
assert [error["loc"] for error in exc_info.value.errors()] == [(field,)]
async def _list_on_vllm_router() -> list[LiteLLMBatch]:
async def _list_on_vllm_router() -> Sequence[Batch]:
page: Final = await litellm.alist_batches(
custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
)
return page.data
async def _list_on_vllm_router_sync() -> list[LiteLLMBatch]:
async def _list_on_vllm_router_sync() -> Sequence[Batch]:
return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data
@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync))
async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[list[LiteLLMBatch]]]
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]]
):
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
200,
@ -1331,3 +1332,38 @@ async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
("batch_5f8dac", "batch", "validating"),
("batch_c74802", "batch", "in_progress"),
]
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
respx_mock.get("http://openai.test/v1/batches").respond(
200,
json={
"object": "list",
"data": [
{
**_OPENAI_BATCH_JSON,
"status": "completed",
"usage": {
"input_tokens": 10,
"input_tokens_details": {"cached_tokens": 4},
"output_tokens": 5,
"output_tokens_details": {"reasoning_tokens": 2},
"total_tokens": 15,
},
},
{**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"},
],
"first_id": "batch_abc123",
"last_id": "batch_def456",
"has_more": False,
},
)
page: Final = await litellm.alist_batches(
custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1"
)
first, second = page.data
assert first.usage is not None
assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2)
assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued")