mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(batches): keep batch list items as OpenAI's Batch so an odd item never fails a list
This commit is contained in:
parent
1d6527c7ac
commit
b6159e2a15
3 changed files with 55 additions and 10 deletions
|
|
@ -34,6 +34,7 @@ from litellm.litellm_core_utils.logging_utils import speech_request_body, track_
|
|||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator, MockResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.types.utils import (
|
||||
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS,
|
||||
EmbeddingResponse,
|
||||
ImageResponse,
|
||||
LiteLLMBatch,
|
||||
|
|
@ -2161,8 +2162,16 @@ class OpenAIFilesAPI(BaseLLM):
|
|||
return response
|
||||
|
||||
|
||||
def _as_litellm_batches(batches: Sequence[Batch]) -> Sequence[LiteLLMBatch]:
|
||||
return [LiteLLMBatch.model_validate(batch.model_dump()) for batch in batches]
|
||||
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
|
||||
return [
|
||||
batch.model_copy(
|
||||
update={
|
||||
"object": batch.object or "batch",
|
||||
"status": OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(batch.status, batch.status),
|
||||
}
|
||||
)
|
||||
for batch in batches
|
||||
]
|
||||
|
||||
|
||||
class OpenAIBatchesAPI(BaseLLM):
|
||||
|
|
@ -2355,7 +2364,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
):
|
||||
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
|
||||
response: Final = await openai_client.batches.list(after=after, limit=limit)
|
||||
return response.model_copy(update={"data": _as_litellm_batches(response.data)})
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
def list_batches(
|
||||
self,
|
||||
|
|
@ -2394,7 +2403,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
if not isinstance(openai_client, OpenAI):
|
||||
raise TypeError("OpenAI client is not an instance of OpenAI. Make sure you passed a sync OpenAI client.")
|
||||
response: Final = openai_client.batches.list(after=after, limit=limit)
|
||||
return response.model_copy(update={"data": _as_litellm_batches(response.data)})
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
|
||||
class OpenAIAssistantsAPI(BaseLLM):
|
||||
|
|
|
|||
|
|
@ -4785,7 +4785,7 @@ class LiteLLMFineTuningJob(FineTuningJob):
|
|||
|
||||
_BatchFieldValue: TypeAlias = object
|
||||
|
||||
_OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
|
||||
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
|
||||
{"pending": "validating", "running": "in_progress"}
|
||||
)
|
||||
|
||||
|
|
@ -4803,7 +4803,7 @@ class LiteLLMBatch(Batch):
|
|||
@field_validator("status", mode="before")
|
||||
@classmethod
|
||||
def _map_vllm_router_status(cls, value: _BatchFieldValue) -> _BatchFieldValue:
|
||||
return _OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value
|
||||
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, _BatchFieldValue]: # mutable-ok: API requires mutation
|
||||
|
|
|
|||
|
|
@ -37,13 +37,14 @@ import litellm.batches.main as bm
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from typing import Final
|
||||
import httpx
|
||||
import respx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from openai.types.batch import Batch
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
|
||||
|
|
@ -1296,20 +1297,20 @@ async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected(
|
|||
assert [error["loc"] for error in exc_info.value.errors()] == [(field,)]
|
||||
|
||||
|
||||
async def _list_on_vllm_router() -> list[LiteLLMBatch]:
|
||||
async def _list_on_vllm_router() -> Sequence[Batch]:
|
||||
page: Final = await litellm.alist_batches(
|
||||
custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
|
||||
)
|
||||
return page.data
|
||||
|
||||
|
||||
async def _list_on_vllm_router_sync() -> list[LiteLLMBatch]:
|
||||
async def _list_on_vllm_router_sync() -> Sequence[Batch]:
|
||||
return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync))
|
||||
async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
|
||||
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[list[LiteLLMBatch]]]
|
||||
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]]
|
||||
):
|
||||
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
|
||||
200,
|
||||
|
|
@ -1331,3 +1332,38 @@ async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
|
|||
("batch_5f8dac", "batch", "validating"),
|
||||
("batch_c74802", "batch", "in_progress"),
|
||||
]
|
||||
|
||||
|
||||
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
|
||||
respx_mock.get("http://openai.test/v1/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
**_OPENAI_BATCH_JSON,
|
||||
"status": "completed",
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"input_tokens_details": {"cached_tokens": 4},
|
||||
"output_tokens": 5,
|
||||
"output_tokens_details": {"reasoning_tokens": 2},
|
||||
"total_tokens": 15,
|
||||
},
|
||||
},
|
||||
{**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"},
|
||||
],
|
||||
"first_id": "batch_abc123",
|
||||
"last_id": "batch_def456",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
page: Final = await litellm.alist_batches(
|
||||
custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1"
|
||||
)
|
||||
|
||||
first, second = page.data
|
||||
assert first.usage is not None
|
||||
assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2)
|
||||
assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue