mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
fix(batches): run hosted_vllm batches in LiteLLM only when the server has no Files API
This commit is contained in:
mateo-berri
parent
3fd964388c
commit
9e8b686c7a
6 changed files with 283 additions and 22 deletions
|
|
@ -24,7 +24,7 @@ from litellm.proxy.batches_endpoints.litellm_executed_batches import (
|
|||
LiteLLMExecutedBatchRunner,
|
||||
ManagedBatchStore,
|
||||
batch_error,
|
||||
litellm_executed_provider_of,
|
||||
litellm_executed_provider_for,
|
||||
resolve_litellm_executed_provider,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import (
|
||||
|
|
@ -95,8 +95,8 @@ def _litellm_executed_batch_runner(llm_router: Router, proxy_logging_obj: ProxyL
|
|||
)
|
||||
|
||||
|
||||
def _raise_when_input_file_must_be_managed(model: str, credentials: Mapping[str, object]) -> None:
|
||||
if litellm_executed_provider_of(credentials) is None:
|
||||
async def _raise_when_input_file_must_be_managed(model: str, credentials: Mapping[str, object]) -> None:
|
||||
if await litellm_executed_provider_for(credentials) is None:
|
||||
return
|
||||
raise batch_error(
|
||||
400,
|
||||
|
|
@ -364,7 +364,9 @@ async def create_batch(
|
|||
detail={"error": "LLM Router not initialized. Ensure models added to proxy."},
|
||||
)
|
||||
|
||||
executed_provider: Final = resolve_litellm_executed_provider(llm_router, model, user_api_key_dict.team_id)
|
||||
executed_provider: Final = await resolve_litellm_executed_provider(
|
||||
llm_router, model, user_api_key_dict.team_id
|
||||
)
|
||||
response = (
|
||||
await _litellm_executed_batch_runner(llm_router, proxy_logging_obj).create(
|
||||
create_request=_create_batch_data,
|
||||
|
|
@ -395,7 +397,7 @@ async def create_batch(
|
|||
model_id=model_param,
|
||||
operation_context="batch creation",
|
||||
)
|
||||
_raise_when_input_file_must_be_managed(model_param, credentials)
|
||||
await _raise_when_input_file_must_be_managed(model_param, credentials)
|
||||
|
||||
prepare_data_with_credentials(
|
||||
data=_create_batch_data,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from itertools import pairwise
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable
|
||||
|
||||
import httpx
|
||||
from openai.types.batch import Errors
|
||||
from openai.types.batch_error import BatchError
|
||||
from openai.types.batch_request_counts import BatchRequestCounts
|
||||
|
|
@ -21,6 +22,7 @@ from litellm.integrations.prometheus import PrometheusLogger
|
|||
from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME
|
||||
from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend
|
||||
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.models.managed_files import LiteLLM_ManagedFileTable
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
|
|
@ -33,7 +35,7 @@ from litellm.proxy.openai_files_endpoints.storage_backend_service import Storage
|
|||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.table_repositories import ManagedObjectRepository
|
||||
from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose
|
||||
from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch
|
||||
from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import models as prisma_models
|
||||
|
|
@ -46,6 +48,7 @@ BatchStatus: TypeAlias = Literal["in_progress", "finalizing", "completed", "fail
|
|||
TERMINAL_BATCH_STATUSES: Final[frozenset[str]] = frozenset({"completed", "failed", "cancelled", "expired"})
|
||||
_BATCH_ENDPOINT_ADAPTER: Final[TypeAdapter[BatchEndpoint]] = TypeAdapter(BatchEndpoint)
|
||||
_CANCEL_POLL_SECONDS: Final = 1.0
|
||||
_FILES_API_PROBE_TIMEOUT_SECONDS: Final = 5.0
|
||||
_COMPLETION_WINDOW_SECONDS: Final = 24 * 60 * 60
|
||||
LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE: Final = (
|
||||
"upload it through POST /v1/files with purpose=batch and either the x-litellm-model header or the "
|
||||
|
|
@ -184,9 +187,67 @@ def litellm_executed_provider_of(credentials: Mapping[str, object]) -> str | Non
|
|||
return provider if provider in LITELLM_EXECUTED_BATCH_PROVIDERS else None
|
||||
|
||||
|
||||
def resolve_litellm_executed_provider(llm_router: "Router", model: str, team_id: str | None) -> str | None:
|
||||
class _HttpGetter(Protocol):
|
||||
async def get(
|
||||
self, url: str, *, headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None
|
||||
) -> httpx.Response: ...
|
||||
|
||||
|
||||
class FilesApiProbe(Protocol):
|
||||
async def __call__(self, api_base: str, api_key: str | None) -> bool: ...
|
||||
|
||||
|
||||
async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_client: _HttpGetter | None = None) -> bool:
|
||||
client: Final = http_client or get_async_httpx_client(llm_provider=LlmProviders.HOSTED_VLLM)
|
||||
try:
|
||||
response: Final = await client.get(
|
||||
f"{api_base.rstrip('/')}/files",
|
||||
headers={"Authorization": f"Bearer {api_key}"} if api_key else None,
|
||||
timeout=_FILES_API_PROBE_TIMEOUT_SECONDS,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
return response.status_code == httpx.codes.NOT_FOUND
|
||||
|
||||
|
||||
def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
|
||||
model: Final = credentials.get("model")
|
||||
api_base: Final = credentials.get("api_base")
|
||||
api_key: Final = credentials.get("api_key")
|
||||
if not isinstance(model, str):
|
||||
return None
|
||||
try:
|
||||
_, _, resolved_api_key, resolved_api_base = litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=provider,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
api_key=api_key if isinstance(api_key, str) else None,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # get_llm_provider raises on a model it cannot map, which means nothing to probe
|
||||
return None
|
||||
return None if resolved_api_base is None else (resolved_api_base, resolved_api_key)
|
||||
|
||||
|
||||
async def litellm_executed_provider_for(
|
||||
credentials: Mapping[str, object], lacks_files_api: FilesApiProbe = upstream_lacks_files_api
|
||||
) -> str | None:
|
||||
provider: Final = litellm_executed_provider_of(credentials)
|
||||
if provider is None:
|
||||
return None
|
||||
upstream: Final = _upstream_of(credentials, provider)
|
||||
if upstream is None:
|
||||
return None
|
||||
return provider if await lacks_files_api(*upstream) else None
|
||||
|
||||
|
||||
async def resolve_litellm_executed_provider(
|
||||
llm_router: "Router",
|
||||
model: str,
|
||||
team_id: str | None,
|
||||
lacks_files_api: FilesApiProbe = upstream_lacks_files_api,
|
||||
) -> str | None:
|
||||
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=model, team_id=team_id)
|
||||
return None if credentials is None else litellm_executed_provider_of(credentials)
|
||||
return None if credentials is None else await litellm_executed_provider_for(credentials, lacks_files_api)
|
||||
|
||||
|
||||
def _provider_of(model: object) -> str | None:
|
||||
|
|
|
|||
|
|
@ -102,24 +102,35 @@ from litellm.types.llms.openai import (
|
|||
router: Final = APIRouter()
|
||||
|
||||
|
||||
def _litellm_executed_batch_input_model(
|
||||
async def _litellm_executed_batch_input_model(
|
||||
llm_router: Router | None,
|
||||
purpose: OpenAIFilesPurpose,
|
||||
model: str | None,
|
||||
target_model_names_list: Sequence[str],
|
||||
team_id: str | None,
|
||||
) -> str | None:
|
||||
if purpose != "batch" or llm_router is None:
|
||||
if llm_router is None:
|
||||
return None
|
||||
candidates: Final = (model,) if model is not None else tuple(target_model_names_list)
|
||||
providers: Final = await asyncio.gather(
|
||||
*(resolve_litellm_executed_provider(llm_router, candidate, team_id) for candidate in candidates)
|
||||
)
|
||||
executed: Final = tuple(
|
||||
candidate
|
||||
for candidate in candidates
|
||||
if resolve_litellm_executed_provider(llm_router, candidate, team_id) is not None
|
||||
candidate for candidate, provider in zip(candidates, providers, strict=True) if provider is not None
|
||||
)
|
||||
match executed:
|
||||
case ():
|
||||
return None
|
||||
case _ if purpose != "batch":
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"The server behind {', '.join(executed)} has no Files API, so LiteLLM keeps only batch input "
|
||||
f"files for it and runs the batch itself: upload with purpose=batch; got purpose={purpose}"
|
||||
),
|
||||
type="invalid_request_error",
|
||||
param="purpose",
|
||||
code=400,
|
||||
)
|
||||
case (only,) if len(candidates) == 1:
|
||||
return only
|
||||
case _:
|
||||
|
|
@ -279,7 +290,7 @@ async def route_create_file(
|
|||
5. Else -> use custom_llm_provider with files_settings
|
||||
"""
|
||||
|
||||
executed_model: Final = _litellm_executed_batch_input_model(
|
||||
executed_model: Final = await _litellm_executed_batch_input_model(
|
||||
llm_router, purpose, model, target_model_names_list, user_api_key_dict.team_id
|
||||
)
|
||||
explicit_storage: Final = target_storage if target_storage and target_storage != "default" else None
|
||||
|
|
|
|||
|
|
@ -37,7 +37,9 @@ from dataclasses import dataclass
|
|||
from typing import Any, Dict, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
|
||||
import litellm
|
||||
|
|
@ -152,6 +154,7 @@ class Harness:
|
|||
router: MagicMock
|
||||
logging: MagicMock
|
||||
creds_resolver: MagicMock
|
||||
upstream_files_route: respx.Route
|
||||
|
||||
@property
|
||||
def router_acreate(self) -> AsyncMock:
|
||||
|
|
@ -174,7 +177,7 @@ def _creds_lookup(*, model_id: str, team_id: str | None = None) -> dict[str, str
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def harness():
|
||||
def harness(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Seam harness. Patches only true I/O boundaries; pure encode/decode/merge
|
||||
helpers run for real. Object mocks are spec'd so unknown method calls raise."""
|
||||
body_holder: Dict[str, Any] = {}
|
||||
|
|
@ -194,6 +197,7 @@ def harness():
|
|||
provider_from_headers = MagicMock(return_value=None)
|
||||
is_known_model = MagicMock(return_value=False)
|
||||
litellm_acreate = AsyncMock(return_value=make_batch())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(endpoints, "_read_request_body", read_body))
|
||||
|
|
@ -215,6 +219,10 @@ def harness():
|
|||
stack.enter_context(patch.object(endpoints, "is_known_model", is_known_model))
|
||||
stack.enter_context(patch.object(litellm, "acreate_batch", litellm_acreate))
|
||||
stack.enter_context(patch.object(litellm, "enable_loadbalancing_on_batch_endpoints", False))
|
||||
upstream = stack.enter_context(respx.mock(assert_all_called=False))
|
||||
upstream_files_route = upstream.get(f"{CREDS['my-vllm']['api_base']}/files").mock(
|
||||
return_value=httpx.Response(404, json={"detail": "Not Found"})
|
||||
)
|
||||
stack.enter_context(patch.object(proxy_server, "llm_router", router))
|
||||
stack.enter_context(patch.object(proxy_server, "proxy_logging_obj", logging))
|
||||
stack.enter_context(patch.object(proxy_server, "general_settings", {}))
|
||||
|
|
@ -233,6 +241,7 @@ def harness():
|
|||
router=router,
|
||||
logging=logging,
|
||||
creds_resolver=router.get_deployment_credentials_with_provider,
|
||||
upstream_files_route=upstream_files_route,
|
||||
)
|
||||
yield h
|
||||
|
||||
|
|
@ -843,6 +852,25 @@ async def test_create__unified_executed_provider_without_database_400(harness):
|
|||
harness.litellm_acreate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__unified_executed_provider_with_its_own_files_api_goes_to_the_provider(harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
harness.upstream_files_route.mock(return_value=httpx.Response(200, json={"object": "list", "data": []}))
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": _managed_input_file_id("my-vllm"),
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
},
|
||||
)
|
||||
await call_create(harness)
|
||||
|
||||
factory.assert_not_called()
|
||||
runner.create.assert_not_called()
|
||||
assert harness.router_kwargs()["model"] == "my-vllm"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__unified_provider_model_never_touches_executed_runner(harness, executed_runner):
|
||||
runner, factory = executed_runner
|
||||
|
|
@ -879,6 +907,26 @@ async def test_create__raw_file_with_executed_model_400_with_upload_guidance(har
|
|||
harness.router_acreate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"upstream_answer",
|
||||
[httpx.Response(200, json={"object": "list", "data": []}), httpx.Response(405), httpx.ConnectError("refused")],
|
||||
ids=["lists files", "files route without list", "unreachable"],
|
||||
)
|
||||
async def test_create__raw_file_with_executed_model_is_forwarded_unless_the_server_lacks_a_files_api(
|
||||
harness, upstream_answer
|
||||
):
|
||||
harness.upstream_files_route.mock(side_effect=[upstream_answer])
|
||||
set_body(harness, {"input_file_id": "file-plain", "endpoint": "/v1/chat/completions", "completion_window": "24h"})
|
||||
|
||||
await call_create(harness, headers={"x-litellm-model": "my-vllm"})
|
||||
|
||||
forwarded = harness.acreate_kwargs()
|
||||
assert forwarded["input_file_id"] == "file-plain"
|
||||
assert forwarded["custom_llm_provider"] == "hosted_vllm"
|
||||
assert forwarded["api_base"] == CREDS["my-vllm"]["api_base"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__model_encoded_beats_unified(harness):
|
||||
"""Precedence row: a file id that is BOTH model-encoded and (pretend) unified
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from dataclasses import dataclass
|
|||
from typing import Final, Literal, cast
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
|
||||
from openai.types.batch_request_counts import BatchRequestCounts
|
||||
|
|
@ -19,9 +20,11 @@ from litellm.proxy.batches_endpoints.litellm_executed_batches import (
|
|||
InvalidBatchInput,
|
||||
LiteLLMExecutedBatchRunner,
|
||||
_resolve_transition,
|
||||
litellm_executed_provider_for,
|
||||
litellm_executed_provider_of,
|
||||
parse_batch_input,
|
||||
resolve_litellm_executed_provider,
|
||||
upstream_lacks_files_api,
|
||||
)
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
_is_base64_encoded_unified_file_id,
|
||||
|
|
@ -424,15 +427,107 @@ def test_litellm_executed_provider_of(credentials: Mapping[str, object], expecte
|
|||
assert litellm_executed_provider_of(credentials) == expected
|
||||
|
||||
|
||||
VLLM_CREDENTIALS: Final[Mapping[str, object]] = {
|
||||
"model": "hosted_vllm/qwen",
|
||||
"api_base": "http://vllm.test/v1/",
|
||||
"api_key": "vllm-key",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FakeFilesApiProbe:
|
||||
lacks_files_api: bool
|
||||
upstreams: list[tuple[str, str | None]]
|
||||
|
||||
async def __call__(self, api_base: str, api_key: str | None) -> bool:
|
||||
self.upstreams.append((api_base, api_key))
|
||||
return self.lacks_files_api
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FakeHttpGetter:
|
||||
outcome: int | httpx.HTTPError
|
||||
requests: list[tuple[str, dict[str, str] | None]]
|
||||
|
||||
async def get(
|
||||
self, url: str, *, headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None
|
||||
) -> httpx.Response:
|
||||
self.requests.append((url, headers))
|
||||
if isinstance(self.outcome, httpx.HTTPError):
|
||||
raise self.outcome
|
||||
return httpx.Response(self.outcome)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("credentials", "expected"), [(None, None), ({"model": "hosted_vllm/qwen"}, "hosted_vllm")], ids=["unknown", "vllm"]
|
||||
("outcome", "expected"),
|
||||
[
|
||||
(404, True),
|
||||
(200, False),
|
||||
(405, False),
|
||||
(401, False),
|
||||
(500, False),
|
||||
(httpx.ConnectError("refused"), False),
|
||||
(httpx.ReadTimeout("slow"), False),
|
||||
],
|
||||
ids=["no files route", "lists files", "files route without list", "unauthorized", "server error", "down", "slow"],
|
||||
)
|
||||
def test_resolve_litellm_executed_provider_asks_the_router_for_the_team_scoped_deployment(
|
||||
async def test_upstream_lacks_files_api_only_when_the_files_route_is_a_404(
|
||||
outcome: int | httpx.HTTPError, expected: bool
|
||||
) -> None:
|
||||
assert await upstream_lacks_files_api("http://vllm.test/v1", "vllm-key", FakeHttpGetter(outcome, [])) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_base", "api_key", "expected_headers"),
|
||||
[
|
||||
("http://vllm.test/v1/", "vllm-key", {"Authorization": "Bearer vllm-key"}),
|
||||
("http://vllm.test/v1", None, None),
|
||||
],
|
||||
ids=["trailing slash with key", "keyless"],
|
||||
)
|
||||
async def test_upstream_lacks_files_api_asks_the_files_route_under_the_api_base(
|
||||
api_base: str, api_key: str | None, expected_headers: dict[str, str] | None
|
||||
) -> None:
|
||||
http_client = FakeHttpGetter(404, [])
|
||||
await upstream_lacks_files_api(api_base, api_key, http_client)
|
||||
assert http_client.requests == [("http://vllm.test/v1/files", expected_headers)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("lacks_files_api", "expected"), [(True, "hosted_vllm"), (False, None)], ids=["bare", "router"]
|
||||
)
|
||||
async def test_litellm_executed_provider_for_leaves_a_server_with_its_own_files_api_alone(
|
||||
lacks_files_api: bool, expected: str | None
|
||||
) -> None:
|
||||
probe = FakeFilesApiProbe(lacks_files_api, [])
|
||||
assert await litellm_executed_provider_for(VLLM_CREDENTIALS, probe) == expected
|
||||
assert probe.upstreams == [("http://vllm.test/v1/", "vllm-key")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"credentials",
|
||||
[{"custom_llm_provider": "openai", "model": "gpt-4o", "api_base": "http://openai.test/v1"}, {"model": 7}],
|
||||
ids=["provider runs its own batches", "no model to resolve an api_base from"],
|
||||
)
|
||||
async def test_litellm_executed_provider_for_never_probes_what_it_would_not_run(
|
||||
credentials: Mapping[str, object],
|
||||
) -> None:
|
||||
probe = FakeFilesApiProbe(True, [])
|
||||
assert await litellm_executed_provider_for(credentials, probe) is None
|
||||
assert probe.upstreams == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("credentials", "expected"), [(None, None), (VLLM_CREDENTIALS, "hosted_vllm")], ids=["unknown", "vllm"]
|
||||
)
|
||||
async def test_resolve_litellm_executed_provider_asks_the_router_for_the_team_scoped_deployment(
|
||||
credentials: Mapping[str, object] | None, expected: str | None
|
||||
) -> None:
|
||||
router = MagicMock(spec=Router)
|
||||
router.get_deployment_credentials_with_provider.return_value = credentials
|
||||
assert resolve_litellm_executed_provider(router, BATCH_MODEL, "team-1") == expected
|
||||
assert (
|
||||
await resolve_litellm_executed_provider(router, BATCH_MODEL, "team-1", FakeFilesApiProbe(True, [])) == expected
|
||||
)
|
||||
router.get_deployment_credentials_with_provider.assert_called_once_with(model_id=BATCH_MODEL, team_id="team-1")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -646,6 +646,7 @@ def batch_upload_seams(mocker: MockerFixture, monkeypatch):
|
|||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router)
|
||||
setup_proxy_logging_object(monkeypatch, llm_router)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test-user"
|
||||
)
|
||||
|
|
@ -666,7 +667,11 @@ def batch_upload_seams(mocker: MockerFixture, monkeypatch):
|
|||
"litellm.acreate_file", new=mocker.AsyncMock(return_value=uploaded)
|
||||
)
|
||||
try:
|
||||
yield stored, provider_upload
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream_files_route = upstream.get("http://vllm.test/v1/files").mock(
|
||||
return_value=httpx.Response(404, json={"detail": "Not Found"})
|
||||
)
|
||||
yield stored, provider_upload, upstream_files_route
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
|
@ -688,7 +693,7 @@ def _upload_batch_file(headers: dict[str, str], form: dict[str, str]):
|
|||
def test_batch_upload_for_a_litellm_executed_model_is_kept_by_litellm(
|
||||
batch_upload_seams, headers: dict[str, str], form: dict[str, str]
|
||||
):
|
||||
stored, provider_upload = batch_upload_seams
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file(headers, form)
|
||||
|
||||
|
|
@ -702,7 +707,7 @@ def test_batch_upload_for_a_litellm_executed_model_is_kept_by_litellm(
|
|||
|
||||
|
||||
def test_batch_upload_naming_an_executed_and_a_provider_model_is_rejected(batch_upload_seams):
|
||||
stored, provider_upload = batch_upload_seams
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file({}, {"target_model_names": "my-vllm,gemini-2.0-flash"})
|
||||
|
||||
|
|
@ -713,8 +718,47 @@ def test_batch_upload_naming_an_executed_and_a_provider_model_is_rejected(batch_
|
|||
provider_upload.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ["assistants", "user_data"])
|
||||
def test_non_batch_upload_for_a_litellm_executed_model_is_rejected_with_the_purpose_to_use(
|
||||
batch_upload_seams, purpose: str
|
||||
):
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file({"x-litellm-model": "my-vllm"}, {"purpose": purpose})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
assert error["param"] == "purpose"
|
||||
assert "purpose=batch" in error["message"]
|
||||
assert f"purpose={purpose}" in error["message"]
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ["batch", "assistants"])
|
||||
@pytest.mark.parametrize(
|
||||
"upstream_answer",
|
||||
[httpx.Response(200, json={"object": "list", "data": []}), httpx.Response(405), httpx.ConnectError("refused")],
|
||||
ids=["lists files", "files route without list", "unreachable"],
|
||||
)
|
||||
def test_upload_for_a_litellm_executed_model_goes_to_the_provider_unless_the_server_lacks_a_files_api(
|
||||
batch_upload_seams, upstream_answer: httpx.Response | httpx.ConnectError, purpose: str
|
||||
):
|
||||
stored, provider_upload, upstream_files_route = batch_upload_seams
|
||||
upstream_files_route.mock(side_effect=[upstream_answer])
|
||||
|
||||
response = _upload_batch_file({"x-litellm-model": "my-vllm"}, {"purpose": purpose})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
stored.assert_not_awaited()
|
||||
provider_upload.assert_awaited_once()
|
||||
assert provider_upload.call_args.kwargs["custom_llm_provider"] == "hosted_vllm"
|
||||
assert provider_upload.call_args.kwargs["api_base"] == "http://vllm.test/v1"
|
||||
|
||||
|
||||
def test_batch_upload_for_a_provider_model_still_goes_to_the_provider(batch_upload_seams):
|
||||
stored, provider_upload = batch_upload_seams
|
||||
stored, provider_upload, _ = batch_upload_seams
|
||||
|
||||
response = _upload_batch_file({"x-litellm-model": "gemini-2.0-flash"}, {})
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue