From 9e8b686c7accbcf8e2cba87b982970dafbad94f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 19 Sep 2026 02:53:46 -0700 Subject: [PATCH] fix(batches): run hosted_vllm batches in LiteLLM only when the server has no Files API --- litellm/proxy/batches_endpoints/endpoints.py | 12 ++- .../litellm_executed_batches.py | 67 +++++++++++- .../openai_files_endpoints/files_endpoints.py | 23 ++-- .../proxy/batches_endpoints/test_endpoints.py | 50 ++++++++- .../test_litellm_executed_batches.py | 101 +++++++++++++++++- .../test_files_endpoint.py | 52 ++++++++- 6 files changed, 283 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 7e49d937171..ba5853b7c46 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -24,7 +24,7 @@ from litellm.proxy.batches_endpoints.litellm_executed_batches import ( LiteLLMExecutedBatchRunner, ManagedBatchStore, batch_error, - litellm_executed_provider_of, + litellm_executed_provider_for, resolve_litellm_executed_provider, ) from litellm.proxy.common_request_processing import ( @@ -95,8 +95,8 @@ def _litellm_executed_batch_runner(llm_router: Router, proxy_logging_obj: ProxyL ) -def _raise_when_input_file_must_be_managed(model: str, credentials: Mapping[str, object]) -> None: - if litellm_executed_provider_of(credentials) is None: +async def _raise_when_input_file_must_be_managed(model: str, credentials: Mapping[str, object]) -> None: + if await litellm_executed_provider_for(credentials) is None: return raise batch_error( 400, @@ -364,7 +364,9 @@ async def create_batch( detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - executed_provider: Final = resolve_litellm_executed_provider(llm_router, model, user_api_key_dict.team_id) + executed_provider: Final = await resolve_litellm_executed_provider( + llm_router, model, user_api_key_dict.team_id + ) response = ( await _litellm_executed_batch_runner(llm_router, proxy_logging_obj).create( create_request=_create_batch_data, @@ -395,7 +397,7 @@ async def create_batch( model_id=model_param, operation_context="batch creation", ) - _raise_when_input_file_must_be_managed(model_param, credentials) + await _raise_when_input_file_must_be_managed(model_param, credentials) prepare_data_with_credentials( data=_create_batch_data, diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index 77e583781ea..642b2ea5f8c 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -7,6 +7,7 @@ from itertools import pairwise from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable +import httpx from openai.types.batch import Errors from openai.types.batch_error import BatchError from openai.types.batch_request_counts import BatchRequestCounts @@ -21,6 +22,7 @@ from litellm.integrations.prometheus import PrometheusLogger from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.models.managed_files import LiteLLM_ManagedFileTable from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup @@ -33,7 +35,7 @@ from litellm.proxy.openai_files_endpoints.storage_backend_service import Storage from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.table_repositories import ManagedObjectRepository from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose -from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch +from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders if TYPE_CHECKING: from prisma import models as prisma_models @@ -46,6 +48,7 @@ BatchStatus: TypeAlias = Literal["in_progress", "finalizing", "completed", "fail TERMINAL_BATCH_STATUSES: Final[frozenset[str]] = frozenset({"completed", "failed", "cancelled", "expired"}) _BATCH_ENDPOINT_ADAPTER: Final[TypeAdapter[BatchEndpoint]] = TypeAdapter(BatchEndpoint) _CANCEL_POLL_SECONDS: Final = 1.0 +_FILES_API_PROBE_TIMEOUT_SECONDS: Final = 5.0 _COMPLETION_WINDOW_SECONDS: Final = 24 * 60 * 60 LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE: Final = ( "upload it through POST /v1/files with purpose=batch and either the x-litellm-model header or the " @@ -184,9 +187,67 @@ def litellm_executed_provider_of(credentials: Mapping[str, object]) -> str | Non return provider if provider in LITELLM_EXECUTED_BATCH_PROVIDERS else None -def resolve_litellm_executed_provider(llm_router: "Router", model: str, team_id: str | None) -> str | None: +class _HttpGetter(Protocol): + async def get( + self, url: str, *, headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None + ) -> httpx.Response: ... + + +class FilesApiProbe(Protocol): + async def __call__(self, api_base: str, api_key: str | None) -> bool: ... + + +async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_client: _HttpGetter | None = None) -> bool: + client: Final = http_client or get_async_httpx_client(llm_provider=LlmProviders.HOSTED_VLLM) + try: + response: Final = await client.get( + f"{api_base.rstrip('/')}/files", + headers={"Authorization": f"Bearer {api_key}"} if api_key else None, + timeout=_FILES_API_PROBE_TIMEOUT_SECONDS, + ) + except httpx.HTTPError: + return False + return response.status_code == httpx.codes.NOT_FOUND + + +def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None: + model: Final = credentials.get("model") + api_base: Final = credentials.get("api_base") + api_key: Final = credentials.get("api_key") + if not isinstance(model, str): + return None + try: + _, _, resolved_api_key, resolved_api_base = litellm.get_llm_provider( + model=model, + custom_llm_provider=provider, + api_base=api_base if isinstance(api_base, str) else None, + api_key=api_key if isinstance(api_key, str) else None, + ) + except Exception: # noqa: BLE001 # get_llm_provider raises on a model it cannot map, which means nothing to probe + return None + return None if resolved_api_base is None else (resolved_api_base, resolved_api_key) + + +async def litellm_executed_provider_for( + credentials: Mapping[str, object], lacks_files_api: FilesApiProbe = upstream_lacks_files_api +) -> str | None: + provider: Final = litellm_executed_provider_of(credentials) + if provider is None: + return None + upstream: Final = _upstream_of(credentials, provider) + if upstream is None: + return None + return provider if await lacks_files_api(*upstream) else None + + +async def resolve_litellm_executed_provider( + llm_router: "Router", + model: str, + team_id: str | None, + lacks_files_api: FilesApiProbe = upstream_lacks_files_api, +) -> str | None: credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=model, team_id=team_id) - return None if credentials is None else litellm_executed_provider_of(credentials) + return None if credentials is None else await litellm_executed_provider_for(credentials, lacks_files_api) def _provider_of(model: object) -> str | None: diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index ad869100fb9..b8dcb89baef 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -102,24 +102,35 @@ from litellm.types.llms.openai import ( router: Final = APIRouter() -def _litellm_executed_batch_input_model( +async def _litellm_executed_batch_input_model( llm_router: Router | None, purpose: OpenAIFilesPurpose, model: str | None, target_model_names_list: Sequence[str], team_id: str | None, ) -> str | None: - if purpose != "batch" or llm_router is None: + if llm_router is None: return None candidates: Final = (model,) if model is not None else tuple(target_model_names_list) + providers: Final = await asyncio.gather( + *(resolve_litellm_executed_provider(llm_router, candidate, team_id) for candidate in candidates) + ) executed: Final = tuple( - candidate - for candidate in candidates - if resolve_litellm_executed_provider(llm_router, candidate, team_id) is not None + candidate for candidate, provider in zip(candidates, providers, strict=True) if provider is not None ) match executed: case (): return None + case _ if purpose != "batch": + raise ProxyException( + message=( + f"The server behind {', '.join(executed)} has no Files API, so LiteLLM keeps only batch input " + f"files for it and runs the batch itself: upload with purpose=batch; got purpose={purpose}" + ), + type="invalid_request_error", + param="purpose", + code=400, + ) case (only,) if len(candidates) == 1: return only case _: @@ -279,7 +290,7 @@ async def route_create_file( 5. Else -> use custom_llm_provider with files_settings """ - executed_model: Final = _litellm_executed_batch_input_model( + executed_model: Final = await _litellm_executed_batch_input_model( llm_router, purpose, model, target_model_names_list, user_api_key_dict.team_id ) explicit_storage: Final = target_storage if target_storage and target_storage != "default" else None diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index e655438a672..3a2ddf50143 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -37,7 +37,9 @@ from dataclasses import dataclass from typing import Any, Dict, Optional from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles import litellm @@ -152,6 +154,7 @@ class Harness: router: MagicMock logging: MagicMock creds_resolver: MagicMock + upstream_files_route: respx.Route @property def router_acreate(self) -> AsyncMock: @@ -174,7 +177,7 @@ def _creds_lookup(*, model_id: str, team_id: str | None = None) -> dict[str, str @pytest.fixture -def harness(): +def harness(monkeypatch: pytest.MonkeyPatch): """Seam harness. Patches only true I/O boundaries; pure encode/decode/merge helpers run for real. Object mocks are spec'd so unknown method calls raise.""" body_holder: Dict[str, Any] = {} @@ -194,6 +197,7 @@ def harness(): provider_from_headers = MagicMock(return_value=None) is_known_model = MagicMock(return_value=False) litellm_acreate = AsyncMock(return_value=make_batch()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) with ExitStack() as stack: stack.enter_context(patch.object(endpoints, "_read_request_body", read_body)) @@ -215,6 +219,10 @@ def harness(): stack.enter_context(patch.object(endpoints, "is_known_model", is_known_model)) stack.enter_context(patch.object(litellm, "acreate_batch", litellm_acreate)) stack.enter_context(patch.object(litellm, "enable_loadbalancing_on_batch_endpoints", False)) + upstream = stack.enter_context(respx.mock(assert_all_called=False)) + upstream_files_route = upstream.get(f"{CREDS['my-vllm']['api_base']}/files").mock( + return_value=httpx.Response(404, json={"detail": "Not Found"}) + ) stack.enter_context(patch.object(proxy_server, "llm_router", router)) stack.enter_context(patch.object(proxy_server, "proxy_logging_obj", logging)) stack.enter_context(patch.object(proxy_server, "general_settings", {})) @@ -233,6 +241,7 @@ def harness(): router=router, logging=logging, creds_resolver=router.get_deployment_credentials_with_provider, + upstream_files_route=upstream_files_route, ) yield h @@ -843,6 +852,25 @@ async def test_create__unified_executed_provider_without_database_400(harness): harness.litellm_acreate.assert_not_called() +@pytest.mark.asyncio +async def test_create__unified_executed_provider_with_its_own_files_api_goes_to_the_provider(harness, executed_runner): + runner, factory = executed_runner + harness.upstream_files_route.mock(return_value=httpx.Response(200, json={"object": "list", "data": []})) + set_body( + harness, + { + "input_file_id": _managed_input_file_id("my-vllm"), + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + ) + await call_create(harness) + + factory.assert_not_called() + runner.create.assert_not_called() + assert harness.router_kwargs()["model"] == "my-vllm" + + @pytest.mark.asyncio async def test_create__unified_provider_model_never_touches_executed_runner(harness, executed_runner): runner, factory = executed_runner @@ -879,6 +907,26 @@ async def test_create__raw_file_with_executed_model_400_with_upload_guidance(har harness.router_acreate.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "upstream_answer", + [httpx.Response(200, json={"object": "list", "data": []}), httpx.Response(405), httpx.ConnectError("refused")], + ids=["lists files", "files route without list", "unreachable"], +) +async def test_create__raw_file_with_executed_model_is_forwarded_unless_the_server_lacks_a_files_api( + harness, upstream_answer +): + harness.upstream_files_route.mock(side_effect=[upstream_answer]) + set_body(harness, {"input_file_id": "file-plain", "endpoint": "/v1/chat/completions", "completion_window": "24h"}) + + await call_create(harness, headers={"x-litellm-model": "my-vllm"}) + + forwarded = harness.acreate_kwargs() + assert forwarded["input_file_id"] == "file-plain" + assert forwarded["custom_llm_provider"] == "hosted_vllm" + assert forwarded["api_base"] == CREDS["my-vllm"]["api_base"] + + @pytest.mark.asyncio async def test_create__model_encoded_beats_unified(harness): """Precedence row: a file id that is BOTH model-encoded and (pretend) unified diff --git a/tests/test_litellm/proxy/batches_endpoints/test_litellm_executed_batches.py b/tests/test_litellm/proxy/batches_endpoints/test_litellm_executed_batches.py index 9c775fd2c97..96e054272a4 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_litellm_executed_batches.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_litellm_executed_batches.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import Final, Literal, cast from unittest.mock import AsyncMock, MagicMock +import httpx import pytest from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles from openai.types.batch_request_counts import BatchRequestCounts @@ -19,9 +20,11 @@ from litellm.proxy.batches_endpoints.litellm_executed_batches import ( InvalidBatchInput, LiteLLMExecutedBatchRunner, _resolve_transition, + litellm_executed_provider_for, litellm_executed_provider_of, parse_batch_input, resolve_litellm_executed_provider, + upstream_lacks_files_api, ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, @@ -424,15 +427,107 @@ def test_litellm_executed_provider_of(credentials: Mapping[str, object], expecte assert litellm_executed_provider_of(credentials) == expected +VLLM_CREDENTIALS: Final[Mapping[str, object]] = { + "model": "hosted_vllm/qwen", + "api_base": "http://vllm.test/v1/", + "api_key": "vllm-key", +} + + +@dataclass(slots=True) +class FakeFilesApiProbe: + lacks_files_api: bool + upstreams: list[tuple[str, str | None]] + + async def __call__(self, api_base: str, api_key: str | None) -> bool: + self.upstreams.append((api_base, api_key)) + return self.lacks_files_api + + +@dataclass(slots=True) +class FakeHttpGetter: + outcome: int | httpx.HTTPError + requests: list[tuple[str, dict[str, str] | None]] + + async def get( + self, url: str, *, headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None + ) -> httpx.Response: + self.requests.append((url, headers)) + if isinstance(self.outcome, httpx.HTTPError): + raise self.outcome + return httpx.Response(self.outcome) + + @pytest.mark.parametrize( - ("credentials", "expected"), [(None, None), ({"model": "hosted_vllm/qwen"}, "hosted_vllm")], ids=["unknown", "vllm"] + ("outcome", "expected"), + [ + (404, True), + (200, False), + (405, False), + (401, False), + (500, False), + (httpx.ConnectError("refused"), False), + (httpx.ReadTimeout("slow"), False), + ], + ids=["no files route", "lists files", "files route without list", "unauthorized", "server error", "down", "slow"], ) -def test_resolve_litellm_executed_provider_asks_the_router_for_the_team_scoped_deployment( +async def test_upstream_lacks_files_api_only_when_the_files_route_is_a_404( + outcome: int | httpx.HTTPError, expected: bool +) -> None: + assert await upstream_lacks_files_api("http://vllm.test/v1", "vllm-key", FakeHttpGetter(outcome, [])) is expected + + +@pytest.mark.parametrize( + ("api_base", "api_key", "expected_headers"), + [ + ("http://vllm.test/v1/", "vllm-key", {"Authorization": "Bearer vllm-key"}), + ("http://vllm.test/v1", None, None), + ], + ids=["trailing slash with key", "keyless"], +) +async def test_upstream_lacks_files_api_asks_the_files_route_under_the_api_base( + api_base: str, api_key: str | None, expected_headers: dict[str, str] | None +) -> None: + http_client = FakeHttpGetter(404, []) + await upstream_lacks_files_api(api_base, api_key, http_client) + assert http_client.requests == [("http://vllm.test/v1/files", expected_headers)] + + +@pytest.mark.parametrize( + ("lacks_files_api", "expected"), [(True, "hosted_vllm"), (False, None)], ids=["bare", "router"] +) +async def test_litellm_executed_provider_for_leaves_a_server_with_its_own_files_api_alone( + lacks_files_api: bool, expected: str | None +) -> None: + probe = FakeFilesApiProbe(lacks_files_api, []) + assert await litellm_executed_provider_for(VLLM_CREDENTIALS, probe) == expected + assert probe.upstreams == [("http://vllm.test/v1/", "vllm-key")] + + +@pytest.mark.parametrize( + "credentials", + [{"custom_llm_provider": "openai", "model": "gpt-4o", "api_base": "http://openai.test/v1"}, {"model": 7}], + ids=["provider runs its own batches", "no model to resolve an api_base from"], +) +async def test_litellm_executed_provider_for_never_probes_what_it_would_not_run( + credentials: Mapping[str, object], +) -> None: + probe = FakeFilesApiProbe(True, []) + assert await litellm_executed_provider_for(credentials, probe) is None + assert probe.upstreams == [] + + +@pytest.mark.parametrize( + ("credentials", "expected"), [(None, None), (VLLM_CREDENTIALS, "hosted_vllm")], ids=["unknown", "vllm"] +) +async def test_resolve_litellm_executed_provider_asks_the_router_for_the_team_scoped_deployment( credentials: Mapping[str, object] | None, expected: str | None ) -> None: router = MagicMock(spec=Router) router.get_deployment_credentials_with_provider.return_value = credentials - assert resolve_litellm_executed_provider(router, BATCH_MODEL, "team-1") == expected + assert ( + await resolve_litellm_executed_provider(router, BATCH_MODEL, "team-1", FakeFilesApiProbe(True, [])) == expected + ) router.get_deployment_credentials_with_provider.assert_called_once_with(model_id=BATCH_MODEL, team_id="team-1") diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 08d7bf8e0da..a8e0097b831 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -646,6 +646,7 @@ def batch_upload_seams(mocker: MockerFixture, monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router) setup_proxy_logging_object(monkeypatch, llm_router) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test-user" ) @@ -666,7 +667,11 @@ def batch_upload_seams(mocker: MockerFixture, monkeypatch): "litellm.acreate_file", new=mocker.AsyncMock(return_value=uploaded) ) try: - yield stored, provider_upload + with respx.mock(assert_all_called=False) as upstream: + upstream_files_route = upstream.get("http://vllm.test/v1/files").mock( + return_value=httpx.Response(404, json={"detail": "Not Found"}) + ) + yield stored, provider_upload, upstream_files_route finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -688,7 +693,7 @@ def _upload_batch_file(headers: dict[str, str], form: dict[str, str]): def test_batch_upload_for_a_litellm_executed_model_is_kept_by_litellm( batch_upload_seams, headers: dict[str, str], form: dict[str, str] ): - stored, provider_upload = batch_upload_seams + stored, provider_upload, _ = batch_upload_seams response = _upload_batch_file(headers, form) @@ -702,7 +707,7 @@ def test_batch_upload_for_a_litellm_executed_model_is_kept_by_litellm( def test_batch_upload_naming_an_executed_and_a_provider_model_is_rejected(batch_upload_seams): - stored, provider_upload = batch_upload_seams + stored, provider_upload, _ = batch_upload_seams response = _upload_batch_file({}, {"target_model_names": "my-vllm,gemini-2.0-flash"}) @@ -713,8 +718,47 @@ def test_batch_upload_naming_an_executed_and_a_provider_model_is_rejected(batch_ provider_upload.assert_not_awaited() +@pytest.mark.parametrize("purpose", ["assistants", "user_data"]) +def test_non_batch_upload_for_a_litellm_executed_model_is_rejected_with_the_purpose_to_use( + batch_upload_seams, purpose: str +): + stored, provider_upload, _ = batch_upload_seams + + response = _upload_batch_file({"x-litellm-model": "my-vllm"}, {"purpose": purpose}) + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "purpose" + assert "purpose=batch" in error["message"] + assert f"purpose={purpose}" in error["message"] + stored.assert_not_awaited() + provider_upload.assert_not_awaited() + + +@pytest.mark.parametrize("purpose", ["batch", "assistants"]) +@pytest.mark.parametrize( + "upstream_answer", + [httpx.Response(200, json={"object": "list", "data": []}), httpx.Response(405), httpx.ConnectError("refused")], + ids=["lists files", "files route without list", "unreachable"], +) +def test_upload_for_a_litellm_executed_model_goes_to_the_provider_unless_the_server_lacks_a_files_api( + batch_upload_seams, upstream_answer: httpx.Response | httpx.ConnectError, purpose: str +): + stored, provider_upload, upstream_files_route = batch_upload_seams + upstream_files_route.mock(side_effect=[upstream_answer]) + + response = _upload_batch_file({"x-litellm-model": "my-vllm"}, {"purpose": purpose}) + + assert response.status_code == 200, response.text + stored.assert_not_awaited() + provider_upload.assert_awaited_once() + assert provider_upload.call_args.kwargs["custom_llm_provider"] == "hosted_vllm" + assert provider_upload.call_args.kwargs["api_base"] == "http://vllm.test/v1" + + def test_batch_upload_for_a_provider_model_still_goes_to_the_provider(batch_upload_seams): - stored, provider_upload = batch_upload_seams + stored, provider_upload, _ = batch_upload_seams response = _upload_batch_file({"x-litellm-model": "gemini-2.0-flash"}, {})