fix: return the provider page for an async client in sync batch list and pass through non-string list statuses

This commit is contained in:
mateo-berri 2026-10-30 12:34:26 -07:00
parent 1ccff0f9ae
commit 4d4cec65b8
4 changed files with 64 additions and 18 deletions

View file

@ -16,6 +16,7 @@ from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._types import RequestOptions
from openai.pagination import SyncCursorPage
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
@ -34,12 +35,12 @@ from litellm.litellm_core_utils.logging_utils import speech_request_body, track_
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator, MockResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.utils import (
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS,
EmbeddingResponse,
ImageResponse,
LiteLLMBatch,
ModelResponse,
ModelResponseStream,
openai_batch_status,
)
from litellm.utils import (
CustomStreamWrapper,
@ -2167,7 +2168,7 @@ def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
batch.model_copy(
update={
"object": batch.object or "batch",
"status": OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(batch.status, batch.status),
"status": openai_batch_status(batch.status),
}
)
for batch in batches
@ -2376,7 +2377,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization: str | None,
after: str | None = None,
limit: int | None = None,
client: OpenAI | None = None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
@ -2400,9 +2401,9 @@ class OpenAIBatchesAPI(BaseLLM):
"OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
)
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
if not isinstance(openai_client, OpenAI):
raise TypeError("OpenAI client is not an instance of OpenAI. Make sure you passed a sync OpenAI client.")
response: Final = openai_client.batches.list(after=after, limit=limit)
if not isinstance(response, SyncCursorPage):
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})

View file

@ -4900,6 +4900,10 @@ OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
)
def openai_batch_status(status: _JsonValue) -> _JsonValue:
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
class LiteLLMBatch(Batch):
_hidden_params: dict = {}
@ -4913,7 +4917,7 @@ class LiteLLMBatch(Batch):
@field_validator("status", mode="before")
@classmethod
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value
return openai_batch_status(value)
@property
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation

View file

@ -1334,6 +1334,25 @@ async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
]
async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string(
respx_mock: respx.MockRouter,
):
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
200,
json={
"object": "list",
"data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}],
"first_id": "batch_5f8dac",
"last_id": "batch_5f8dac",
"has_more": False,
},
)
batches: Final = await _list_on_vllm_router()
assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])]
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
respx_mock.get("http://openai.test/v1/batches").respond(
200,

View file

@ -2051,16 +2051,38 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope
assert [message["role"] for message in messages] == expected_roles
def test_sync_batch_list_refuses_an_async_client_instead_of_returning_an_unawaited_page():
async_client: Final = AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1")
@pytest.mark.asyncio
async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await(
respx_mock: respx.MockRouter,
):
respx_mock.get("http://vllm-router.test/v1/batches").respond(
200,
json={
"object": "list",
"data": [
{
"id": "batch_5f8dac",
"status": "pending",
"input_file_id": "file-b73f4c",
"created_at": 1793239207,
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
}
],
"first_id": "batch_5f8dac",
"last_id": "batch_5f8dac",
"has_more": False,
},
)
with pytest.raises(TypeError, match="not an instance of OpenAI"):
OpenAIBatchesAPI().list_batches(
_is_async=False,
api_key="sk-test",
api_base="http://vllm-router.test/v1",
timeout=5.0,
max_retries=0,
organization=None,
client=async_client,
)
page: Final = OpenAIBatchesAPI().list_batches(
_is_async=False,
api_key="sk-test",
api_base="http://vllm-router.test/v1",
timeout=5.0,
max_retries=0,
organization=None,
client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1"),
)
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]