mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix: return the provider page for an async client in sync batch list and pass through non-string list statuses
This commit is contained in:
parent
1ccff0f9ae
commit
4d4cec65b8
4 changed files with 64 additions and 18 deletions
|
|
@ -16,6 +16,7 @@ from openai._base_client import make_request_options
|
|||
from openai._constants import RAW_RESPONSE_HEADER
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai._types import RequestOptions
|
||||
from openai.pagination import SyncCursorPage
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
from openai.types.beta.assistant_deleted import AssistantDeleted
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
|
|
@ -34,12 +35,12 @@ from litellm.litellm_core_utils.logging_utils import speech_request_body, track_
|
|||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator, MockResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.types.utils import (
|
||||
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS,
|
||||
EmbeddingResponse,
|
||||
ImageResponse,
|
||||
LiteLLMBatch,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
openai_batch_status,
|
||||
)
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
|
|
@ -2167,7 +2168,7 @@ def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
|
|||
batch.model_copy(
|
||||
update={
|
||||
"object": batch.object or "batch",
|
||||
"status": OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(batch.status, batch.status),
|
||||
"status": openai_batch_status(batch.status),
|
||||
}
|
||||
)
|
||||
for batch in batches
|
||||
|
|
@ -2376,7 +2377,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
organization: str | None,
|
||||
after: str | None = None,
|
||||
limit: int | None = None,
|
||||
client: OpenAI | None = None,
|
||||
client: OpenAI | AsyncOpenAI | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
):
|
||||
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
|
||||
|
|
@ -2400,9 +2401,9 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
"OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
|
||||
)
|
||||
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
|
||||
if not isinstance(openai_client, OpenAI):
|
||||
raise TypeError("OpenAI client is not an instance of OpenAI. Make sure you passed a sync OpenAI client.")
|
||||
response: Final = openai_client.batches.list(after=after, limit=limit)
|
||||
if not isinstance(response, SyncCursorPage):
|
||||
return response
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4900,6 +4900,10 @@ OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
def openai_batch_status(status: _JsonValue) -> _JsonValue:
|
||||
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
|
||||
|
||||
|
||||
class LiteLLMBatch(Batch):
|
||||
_hidden_params: dict = {}
|
||||
|
||||
|
|
@ -4913,7 +4917,7 @@ class LiteLLMBatch(Batch):
|
|||
@field_validator("status", mode="before")
|
||||
@classmethod
|
||||
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
|
||||
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(value, value) if isinstance(value, str) else value
|
||||
return openai_batch_status(value)
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation
|
||||
|
|
|
|||
|
|
@ -1334,6 +1334,25 @@ async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
|
|||
]
|
||||
|
||||
|
||||
async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_5f8dac",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
batches: Final = await _list_on_vllm_router()
|
||||
|
||||
assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])]
|
||||
|
||||
|
||||
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
|
||||
respx_mock.get("http://openai.test/v1/batches").respond(
|
||||
200,
|
||||
|
|
|
|||
|
|
@ -2051,16 +2051,38 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope
|
|||
assert [message["role"] for message in messages] == expected_roles
|
||||
|
||||
|
||||
def test_sync_batch_list_refuses_an_async_client_instead_of_returning_an_unawaited_page():
|
||||
async_client: Final = AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1")
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
respx_mock.get("http://vllm-router.test/v1/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "batch_5f8dac",
|
||||
"status": "pending",
|
||||
"input_file_id": "file-b73f4c",
|
||||
"created_at": 1793239207,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
}
|
||||
],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_5f8dac",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(TypeError, match="not an instance of OpenAI"):
|
||||
OpenAIBatchesAPI().list_batches(
|
||||
_is_async=False,
|
||||
api_key="sk-test",
|
||||
api_base="http://vllm-router.test/v1",
|
||||
timeout=5.0,
|
||||
max_retries=0,
|
||||
organization=None,
|
||||
client=async_client,
|
||||
)
|
||||
page: Final = OpenAIBatchesAPI().list_batches(
|
||||
_is_async=False,
|
||||
api_key="sk-test",
|
||||
api_base="http://vllm-router.test/v1",
|
||||
timeout=5.0,
|
||||
max_retries=0,
|
||||
organization=None,
|
||||
client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1"),
|
||||
)
|
||||
|
||||
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue