scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-22 00:31:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 51

fix(batches): shape LiteLLM-executed batch errors like OpenAI errors

Browse source
This commit is contained in:
mateo-berri 2026-09-19 01:58:53 -07:00
parent fd45412c89
commit 167e3244ab
3 changed files with 35 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/proxy/batches_endpoints/endpoints.py
View file

@ -23,7 +23,7 @@ from litellm.proxy.batches_endpoints.litellm_executed_batches import (
LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE,
LiteLLMExecutedBatchRunner,
ManagedBatchStore,
batch_http_error,
batch_error,
litellm_executed_provider_of,
resolve_litellm_executed_provider,
)
@ -83,7 +83,7 @@ def _litellm_executed_batch_runner(llm_router: Router, proxy_logging_obj: ProxyL
managed_files: Final = proxy_logging_obj.get_proxy_hook("managed_files")
if prisma_client is None or not isinstance(managed_files, ManagedBatchStore):
raise batch_http_error(
raise batch_error(
400,
"LiteLLM-executed batches need a database: set DATABASE_URL so LiteLLM can keep the batch and its files",
)
@ -98,7 +98,7 @@ def _litellm_executed_batch_runner(llm_router: Router, proxy_logging_obj: ProxyL
def _raise_when_input_file_must_be_managed(model: str, credentials: Mapping[str, object]) -> None:
if litellm_executed_provider_of(credentials) is None:
return
raise batch_http_error(
raise batch_error(
400,
f"Batches for {model} run inside LiteLLM, so the input file must be a LiteLLM managed file: "
f"{LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE}",
@ -556,7 +556,7 @@ async def retrieve_batch(
executed_batch: Final = isinstance(unified_batch_id, str) and is_litellm_executed_batch(unified_batch_id)
if executed_batch and response is None:
raise batch_http_error(404, f"No batch found with id '{batch_id}'.")
raise batch_error(404, f"No batch found with id '{batch_id}'.")
# If batch is in a terminal state, return immediately.
# Include "complete" (DB-normalized form of "completed").
@ -1067,7 +1067,7 @@ async def cancel_batch(
# SCENARIO 2: target_model_names based routing
elif unified_batch_id and is_litellm_executed_batch(unified_batch_id):
if llm_router is None:
raise batch_http_error(500, "LLM Router not initialized. Ensure models added to proxy.")
raise batch_error(500, "LLM Router not initialized. Ensure models added to proxy.")
response = await _litellm_executed_batch_runner( # rebind-ok: each cancel path sets the route's response
llm_router, proxy_logging_obj
).cancel(batch_id, user_api_key_dict)

21
litellm/proxy/batches_endpoints/litellm_executed_batches.py
View file

@ -7,7 +7,6 @@ from itertools import pairwise
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable
from fastapi import HTTPException
from openai.types.batch import Errors
from openai.types.batch_error import BatchError
from openai.types.batch_request_counts import BatchRequestCounts
@ -23,7 +22,7 @@ from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_ST
from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
from litellm.models.managed_files import LiteLLM_ManagedFileTable
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.openai_files_endpoints.common_utils import (
LITELLM_EXECUTED_BATCH_ID_PREFIX,
@ -234,16 +233,16 @@ def parse_batch_input(content: bytes, endpoint: BatchEndpoint) -> tuple[BatchInp
return lines
def batch_http_error(status_code: int, message: str) -> HTTPException:
detail: Final = {"error": message} # mutable-ok: HTTPException detail must be a plain mapping
return HTTPException(status_code=status_code, detail=detail)
def batch_error(status_code: int, message: str) -> ProxyException:
error_type: Final = "invalid_request_error" if status_code < 500 else ProxyErrorTypes.internal_server_error.value
return ProxyException(message=message, type=error_type, param=None, code=status_code)
def _validate_endpoint(endpoint: object) -> BatchEndpoint:
try:
return _BATCH_ENDPOINT_ADAPTER.validate_python(endpoint)
except ValidationError:
raise batch_http_error(400, f"endpoint {endpoint!r} is not supported for a LiteLLM-executed batch")
raise batch_error(400, f"endpoint {endpoint!r} is not supported for a LiteLLM-executed batch")
def _status_code_of(error: Exception) -> int:
@ -350,7 +349,7 @@ class LiteLLMExecutedBatchRunner:
content: Final = await self._download_input(unified_input_file_id, user_api_key_dict)
parsed: Final = parse_batch_input(content, endpoint)
if isinstance(parsed, InvalidBatchInput):
raise batch_http_error(400, f"Invalid batch input file: {parsed.describe()}")
raise batch_error(400, f"Invalid batch input file: {parsed.describe()}")
llm_batch_id: Final = f"{LITELLM_EXECUTED_BATCH_ID_PREFIX}{uuid_module.uuid4().hex}"
model_id: Final = next(iter(self.llm_router.get_model_ids(model_name=model)), model)
unified_batch_id: Final = self.managed_files.get_unified_batch_id(batch_id=llm_batch_id, model_id=model_id)
@ -397,9 +396,9 @@ class LiteLLMExecutedBatchRunner:
async def cancel(self, unified_batch_id: str, user_api_key_dict: UserAPIKeyAuth) -> LiteLLMBatch:
current: Final = await self._load_batch(unified_batch_id)
if current is None:
raise batch_http_error(404, f"Batch {unified_batch_id} not found")
raise batch_error(404, f"Batch {unified_batch_id} not found")
if current.status in TERMINAL_BATCH_STATUSES:
raise batch_http_error(400, f"Cannot cancel a batch with status '{current.status}'")
raise batch_error(400, f"Cannot cancel a batch with status '{current.status}'")
if current.status == "cancelling":
return current
cancelling: Final = current.model_copy(
@ -413,7 +412,7 @@ class LiteLLMExecutedBatchRunner:
unified_input_file_id, litellm_parent_otel_span=user_api_key_dict.parent_otel_span
)
if stored is None or not stored.storage_backend or not stored.storage_url:
raise batch_http_error(
raise batch_error(
400,
f"LiteLLM does not hold the content of input file {unified_input_file_id}: "
f"{LITELLM_EXECUTED_BATCH_UPLOAD_GUIDANCE}",
@ -422,7 +421,7 @@ class LiteLLMExecutedBatchRunner:
backend: Final = self.storage_backend_factory(stored.storage_backend, prisma_client=self.prisma_client)
return await backend.download_file(stored.storage_url)
except ValueError as e:
raise batch_http_error(400, str(e))
raise batch_error(400, str(e))
async def _run(self, run: _BatchRun) -> None:
try:

40
tests/test_litellm/proxy/batches_endpoints/test_litellm_executed_batches.py
View file

@ -6,12 +6,11 @@ from typing import Final, Literal, cast
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import HTTPException
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
from openai.types.batch_request_counts import BatchRequestCounts
from litellm.models.managed_files import LiteLLM_ManagedFileTable
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.batches_endpoints import litellm_executed_batches
from litellm.proxy.batches_endpoints.litellm_executed_batches import (
BatchEndpoint,
@ -547,10 +546,11 @@ async def test_create_splits_failed_rows_into_the_error_file() -> None:
async def test_create_rejects_an_unsupported_endpoint() -> None:
harness = make_runner()
with pytest.raises(HTTPException) as raised:
with pytest.raises(ProxyException) as raised:
await harness.create(endpoint="/v1/moderations")
assert raised.value.status_code == 400
assert "/v1/moderations" in raised.value.detail["error"]
assert raised.value.code == "400"
assert raised.value.type == "invalid_request_error"
assert "/v1/moderations" in raised.value.message
assert harness.store.calls == []
assert harness.storage_factory.calls == []
@ -564,30 +564,30 @@ async def test_create_rejects_an_input_file_litellm_does_not_hold(
files: Mapping[str, LiteLLM_ManagedFileTable],
) -> None:
harness = make_runner(files=files)
with pytest.raises(HTTPException) as raised:
with pytest.raises(ProxyException) as raised:
await harness.create()
assert raised.value.status_code == 400
assert "POST /v1/files" in raised.value.detail["error"]
assert raised.value.code == "400"
assert "POST /v1/files" in raised.value.message
assert harness.storage_factory.calls == []
assert harness.store.calls == []
async def test_create_rejects_an_invalid_input_file() -> None:
harness = make_runner(content=jsonl(chat_row("a", "hi"), chat_row("a", "again")))
with pytest.raises(HTTPException) as raised:
with pytest.raises(ProxyException) as raised:
await harness.create()
assert raised.value.status_code == 400
assert raised.value.detail["error"].startswith("Invalid batch input file:")
assert "'a'" in raised.value.detail["error"]
assert raised.value.code == "400"
assert raised.value.message.startswith("Invalid batch input file:")
assert "'a'" in raised.value.message
assert harness.store.calls == []
async def test_create_surfaces_a_storage_backend_error_as_a_400() -> None:
harness = make_runner(storage_error=ValueError("Unknown storage backend 's3'"))
with pytest.raises(HTTPException) as raised:
with pytest.raises(ProxyException) as raised:
await harness.create()
assert raised.value.status_code == 400
assert raised.value.detail["error"] == "Unknown storage backend 's3'"
assert raised.value.code == "400"
assert raised.value.message == "Unknown storage backend 's3'"
assert harness.store.calls == []
@ -617,18 +617,18 @@ async def test_each_endpoint_awaits_only_its_router_method(
async def test_cancel_unknown_batch_is_404() -> None:
harness = make_runner()
with pytest.raises(HTTPException) as raised:
with pytest.raises(ProxyException) as raised:
await harness.runner.cancel("missing-batch", harness.user)
assert raised.value.status_code == 404
assert raised.value.code == "404"
async def test_cancel_terminal_batch_is_400() -> None:
harness = make_runner()
batch = seeded_batch(harness.store, "completed")
with pytest.raises(HTTPException) as raised:
with pytest.raises(ProxyException) as raised:
await harness.runner.cancel(batch.id, harness.user)
assert raised.value.status_code == 400
assert "completed" in raised.value.detail["error"]
assert raised.value.code == "400"
assert "completed" in raised.value.message
assert harness.store.calls == []