This commit is contained in:
Mateo Wang 2026-10-27 16:03:24 -07:00 • committed by GitHub
commit a1a45a5791
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 702 additions and 122 deletions

View file

@ -42,11 +42,12 @@ if TYPE_CHECKING:
def _error_names_resource(error: Exception, resource_id: str) -> bool:
return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
return resource_id in re.findall(r"""[^\s"'`<>]+""", str(error))
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
UNSETTLED_POLLS_BEFORE_EXPIRY: Final = 3
PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
"completed",
@ -91,6 +92,10 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
return TeamRepository(prisma_client).table
def _unique_jobs(jobs: Sequence[ManagedObjectRow]) -> tuple[ManagedObjectRow, ...]:
return tuple({job.id: job for job in jobs}.values())
class CheckBatchCost:
def __init__(
self,
@ -116,6 +121,7 @@ class CheckBatchCost:
self._arrival_cursor: tuple[datetime, str] | None = None
self._arrival_before: datetime = self._poll_before
self._poll_arrivals: bool = False
self._unsettled_polls: Mapping[str, int] = MappingProxyType({})
self._line_item_stores: Final[set[asyncio.Task[None]]] = set()
self._line_item_store_turn: Final = asyncio.Lock()
self._poll_idle: Final = asyncio.Event()
@ -147,27 +153,6 @@ class CheckBatchCost:
return
self.batch_processed_support_confirmed = True
async def _cleanup_stale_managed_objects(self) -> None:
"""
Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
in non-terminal states as 'stale_expired'. These will never complete and
should not be polled.
"""
cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
result: Final = await _managed_object_table(self.prisma_client).update_many(
where={
"file_purpose": "batch",
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
"created_at": {"lt": cutoff},
},
data={"status": "stale_expired"},
)
if result > 0:
verbose_proxy_logger.warning(
f"CheckBatchCost: marked {result} stale managed objects "
f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
)
async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]":
"""Query batch jobs without the batch_processed filter (for older schemas)."""
return await self._find_jobs(
@ -187,14 +172,37 @@ class CheckBatchCost:
)
async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]:
arrivals: Final = self._poll_arrivals
page: Final = await self._find_job_page(where, arrivals=arrivals)
self._poll_arrivals = not arrivals if page else arrivals
if page:
return page
return await self._find_job_page(where, arrivals=not arrivals)
lead_arrivals: Final = self._poll_arrivals
self._poll_arrivals = not lead_arrivals
lead_share: Final = MAX_OBJECTS_PER_POLL_CYCLE - MAX_OBJECTS_PER_POLL_CYCLE // 2
lead: Final = await self._find_job_page(where, arrivals=lead_arrivals, take=lead_share)
trail: Final = await self._find_job_page(
where, arrivals=not lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead)
)
spill: Final = (
await self._find_job_page(
where, arrivals=lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead) - len(trail)
)
if len(lead) == lead_share
else ()
)
return _unique_jobs((*lead, *trail, *spill))
async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]:
async def _find_job_page(
self, where: Mapping[str, object], *, arrivals: bool, take: int
) -> Sequence[ManagedObjectRow]:
if take == 0:
return ()
resumed_mid_lane: Final = (self._arrival_cursor if arrivals else self._poll_cursor) is not None
page: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take)
if len(page) == take or not resumed_mid_lane:
return page
wrapped: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take - len(page))
return _unique_jobs((*page, *wrapped))
async def _find_job_page_from_cursor(
self, where: Mapping[str, object], *, arrivals: bool, take: int
) -> Sequence[ManagedObjectRow]:
cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
if cursor is None:
if arrivals:
@ -220,10 +228,10 @@ class CheckBatchCost:
else {}
),
},
take=MAX_OBJECTS_PER_POLL_CYCLE,
take=take,
order=[{"created_at": "asc"}, {"id": "asc"}],
)
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == take else None
if arrivals:
self._arrival_cursor = next_cursor
else:
@ -231,11 +239,65 @@ class CheckBatchCost:
return page
def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]:
return {
"id": job.id,
"updated_at": job.updated_at,
**({"batch_processed": False} if self._has_batch_processed_column else {}),
}
if self._has_batch_processed_column:
return {"id": job.id, "batch_processed": False}
return {"id": job.id, "updated_at": job.updated_at}
@staticmethod
def _is_past_staleness_cutoff(job: ManagedObjectRow) -> bool:
return job.created_at < datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
async def _expire_if_stale(self, job: ManagedObjectRow) -> None:
if not self._is_past_staleness_cutoff(job):
return
if job.status in TERMINAL_MANAGED_OBJECT_STATUSES:
await self._retire_job(job, f"unbilled {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days after it was created")
return
await self._expire_unfinished(job)
async def _expire_after_repeated_unsettled_polls(self, job: ManagedObjectRow) -> None:
unsettled: Final = self._unsettled_polls.get(job.id, 0) + 1
if unsettled < UNSETTLED_POLLS_BEFORE_EXPIRY:
self._unsettled_polls = MappingProxyType({**self._unsettled_polls, job.id: unsettled})
return
self._forget_unsettled_polls(job)
await self._expire_if_stale(job)
def _forget_unsettled_polls(self, job: ManagedObjectRow) -> None:
if job.id in self._unsettled_polls:
self._unsettled_polls = MappingProxyType(
{job_id: count for job_id, count in self._unsettled_polls.items() if job_id != job.id}
)
@staticmethod
def _is_completed_with_output_id_pending(response: "LiteLLMBatch") -> bool:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
return (
response.status in ("completed", "complete")
and response.output_file_id is None
and not _completed_batch_safe_to_retire(response)
)
async def _expire_unfinished(self, job: ManagedObjectRow) -> None:
try:
expired: Final = await _managed_object_table(self.prisma_client).update_many(
where={
**self._unprocessed_snapshot(job),
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
},
data={"status": "stale_expired"},
)
except Exception as db_err:
verbose_proxy_logger.error(f"CheckBatchCost: failed to mark stale job {job.id} as stale_expired: {db_err}")
return
if expired > 0:
verbose_proxy_logger.warning(
f"CheckBatchCost: job {job.id} is older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days "
"and still not finished, so it is marked stale_expired and no longer polled"
)
async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None:
"""
@ -345,6 +407,16 @@ class CheckBatchCost:
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id)
@staticmethod
def _is_lasting_provider_failure(error: Exception) -> bool:
import openai
from litellm.llms.base_llm.chat.transformation import BaseLLMException
return isinstance(error, (BaseLLMException, openai.APIStatusError)) and not litellm.should_retry(
error.status_code
)
def _batch_deployment_exists(self, model_id: str) -> bool:
"""A 404 only proves the batch is gone when it came from the batch's own
deployment. Once that deployment leaves the router, default fallbacks can
@ -356,15 +428,21 @@ class CheckBatchCost:
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
"""A 404 naming the output file means there is nothing to fetch on this or any
later poll: providers like Vertex AI advertise an output path for every batch,
including terminal ones that never wrote it. Any other failure may be
transient, so it keeps retrying until the staleness sweep bounds it."""
including terminal ones that never wrote it. GCS names a gs:// object without its scheme,
and S3 names an s3:// object by its key alone. Any other failure keeps retrying."""
import openai
from litellm.exceptions import NotFoundError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
if not output_file_id:
return False
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
is_not_found: Final = isinstance(error, (NotFoundError, openai.NotFoundError)) or (
isinstance(error, BaseLLMException) and error.status_code == 404
)
object_path: Final = output_file_id.split("://", 1)[-1]
names: Final = (output_file_id, object_path, object_path.split("/", 1)[-1])
return is_not_found and any(_error_names_resource(error, name) for name in names)
async def _finalize_unbilled_terminal_job(
self, job: "ManagedObjectRow", response: "LiteLLMBatch"
@ -1094,13 +1172,6 @@ class CheckBatchCost:
processed_models: List[Tuple[Optional[str], Optional[str]]] = []
try:
await self._cleanup_stale_managed_objects()
except Exception as cleanup_err:
verbose_proxy_logger.warning(
f"CheckBatchCost: stale cleanup failed (poll will continue): {cleanup_err}"
)
# Look for all batches that have not yet been processed by CheckBatchCost.
# self._has_batch_processed_column is cached after the first probe so that
# older schemas don't pay a guaranteed-failing primary query + warning on
@ -1142,6 +1213,8 @@ class CheckBatchCost:
if routing is None:
if self._has_unified_id_without_model(job):
await self._retire_job(job, "unified object id has no model id")
else:
await self._expire_if_stale(job)
continue
model_id, batch_id = routing
@ -1166,7 +1239,22 @@ class CheckBatchCost:
prom_logger.record_check_batch_cost_error("provider_retrieval_error")
if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id):
await self._retire_job(job, f"batch {batch_id} no longer exists at the provider")
elif (
self._is_past_staleness_cutoff(job)
and job.status not in TERMINAL_MANAGED_OBJECT_STATUSES
and self._is_lasting_provider_failure(e)
):
await self._expire_after_repeated_unsettled_polls(job)
continue
if self._is_completed_with_output_id_pending(response):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
if self._is_past_staleness_cutoff(job):
await self._expire_after_repeated_unsettled_polls(job)
continue
self._forget_unsettled_polls(job)
## RETRIEVE THE BATCH JOB OUTPUT FILE
if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
@ -1217,16 +1305,6 @@ class CheckBatchCost:
self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
continue
await self._retire_unbilled_terminal_job(job, response, model_id, batch_id)
# Record polling run metrics (always, even if nothing was processed)

View file

@ -6,6 +6,7 @@ import base64
import json
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from datetime import datetime, timedelta, timezone
from itertools import chain
from types import MappingProxyType
from typing import (
@ -39,6 +40,7 @@ from litellm.constants import (
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS,
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_FILE_LIST_LIMIT,
PENDING_RESPONSE_STATUSES,
)
@ -2387,12 +2389,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
where={
"file_purpose": "batch",
"batch_processed": False,
"status": {"not_in": ["failed", "expired", "cancelled"]},
"status": {"not_in": ["failed", "expired", "cancelled", "stale_expired"]},
},
take=MAX_MATCHES_TO_RETURN,
order={"created_at": "desc"},
)
input_file_cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
referencing_batches: Final[list[dict[str, object]]] = []
for batch in batches:
try:
@ -2405,7 +2408,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Extract file IDs from batch
# Batches typically reference the unified file ID in input_file_id
# Output and error files are generated by the provider
input_file_id = batch_data.get("input_file_id")
input_file_id = batch_data.get("input_file_id") if batch.created_at >= input_file_cutoff else None
output_file_id = batch_data.get("output_file_id")
error_file_id = batch_data.get("error_file_id")

View file

@ -10,6 +10,7 @@ This ensures cost tracking is not disrupted by premature file deletion.
import base64
import json
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -50,7 +51,7 @@ def _make_batch_db_record(
mock_batch.status = status
mock_batch.file_object = json.dumps(file_object)
mock_batch.created_by = created_by
mock_batch.created_at = 1700000000
mock_batch.created_at = datetime.now(timezone.utc)
return mock_batch

View file

@ -12,19 +12,23 @@ import logging
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from copy import copy
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from hashlib import sha256
from queue import SimpleQueue
from types import SimpleNamespace
from typing import TYPE_CHECKING, Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from fastapi import HTTPException
import litellm
from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS
from litellm.types.router import LiteLLM_Params
from litellm.types.utils import LiteLLMBatch
from litellm_enterprise.proxy.common_utils.check_batch_cost import UNSETTLED_POLLS_BEFORE_EXPIRY
if TYPE_CHECKING:
from litellm.batches.batch_utils import BatchCostUsageResult
@ -154,23 +158,6 @@ class TestCheckBatchCost:
llm_router=mock_llm_router,
)
@pytest.mark.asyncio
async def test_cleanup_scoped_to_batch_file_purpose(self, check_batch_cost_instance, mock_prisma_client):
"""_cleanup_stale_managed_objects scopes its update to file_purpose='batch' only."""
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
# Return empty so the main poll loop exits immediately
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])
await check_batch_cost_instance.check_batch_cost()
calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list
stale_call = calls[0]
assert stale_call[1]["data"] == {"status": "stale_expired"}
where = stale_call[1]["where"]
assert where["file_purpose"] == "batch"
assert "stale_expired" in where["status"]["not_in"]
assert "created_at" in where
@pytest.mark.asyncio
async def test_startup_probe_confirms_batch_processed_support(self, check_batch_cost_instance, mock_prisma_client):
mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None)
@ -251,7 +238,7 @@ class TestCheckBatchCost:
fallback_where = calls[1][1]["where"]
assert "batch_processed" not in fallback_where
assert "stale_expired" in fallback_where["status"]["not_in"]
assert calls[1][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
assert calls[2][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
# Column absence is now cached — next call should go straight to fallback
assert check_batch_cost_instance._has_batch_processed_column is False
assert check_batch_cost_instance.batch_processed_support_confirmed is False
@ -744,7 +731,7 @@ class TestCheckBatchCost:
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
"Expected update() to be called exactly once for the completed job"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
@ -877,7 +864,7 @@ class TestCheckBatchCost:
mock_update_database.assert_awaited_once()
assert mock_update_database.call_args.kwargs["response_cost"] == 0.01
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
"the job must still be marked processed once cost tracking succeeds"
)
@ -979,7 +966,7 @@ class TestCheckBatchCost:
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
f"Expected update() to be called exactly once for a {terminal_status} job"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
@ -1085,13 +1072,9 @@ class TestCheckBatchCost:
assert owner.user_id == "user-1"
assert owner.team_id == "team-1"
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1
update_call = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args
assert update_call.kwargs["where"] == {
"id": "job-terminal-mint-1",
"batch_processed": False,
"updated_at": mock_job.updated_at,
}
assert update_call.kwargs["where"] == {"id": "job-terminal-mint-1", "batch_processed": False}
update_data = update_call.kwargs["data"]
assert update_data["status"] == terminal_status
assert update_data["batch_processed"] is True
@ -1156,7 +1139,7 @@ class TestCheckBatchCost:
) as mock_afile_content:
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
"a completed batch with no output file must be marked processed exactly once"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
@ -1205,6 +1188,7 @@ class TestCheckBatchCost:
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
mock_job.created_by = "user-1"
mock_job.created_at = datetime.now(timezone.utc)
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
@ -1245,6 +1229,7 @@ class TestCheckBatchCost:
mock_job.id = "job-in-progress-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
mock_job.created_at = datetime.now(timezone.utc)
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
@ -1375,7 +1360,7 @@ class TestCheckBatchCost:
f"{terminal_status} batch with an output file must fetch results and be billed"
)
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True
assert update_data["status"] == terminal_status, (
@ -1548,7 +1533,7 @@ class TestCheckBatchCost:
assert mock_afile_content.await_count == 1
mock_calculate.assert_not_awaited()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
"a terminal batch with a 404ing output file must be retired, not retried forever"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
@ -1946,7 +1931,7 @@ class TestUnmanagedVertexRouting:
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 1
update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True
assert update_data["status"] == "complete"
@ -2163,7 +2148,7 @@ class TestUnmanagedBedrockRouting:
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 1
update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True
assert update_data["status"] == "complete"
@ -2725,6 +2710,7 @@ class TestPollPageStarvation:
job.id = job_id
job.unified_object_id = unified_object_id
job.created_by = "user-1"
job.created_at = datetime.now(timezone.utc)
return job
@staticmethod
@ -2910,6 +2896,168 @@ class TestPollPageStarvation:
prisma.db.litellm_managedobjecttable.update.assert_not_awaited()
@pytest.mark.asyncio
async def test_a_new_batch_is_polled_every_cycle_while_an_older_batch_stays_unbilled(self):
"""An unbilled batch older than the arrival window used to take every other cycle away from the arrival
lane, so a new batch waited two poll intervals to be billed."""
now: Final = datetime.now(timezone.utc)
rows: Final = tuple(
SimpleNamespace(
id=f"job-{name}",
unified_object_id=self._encode(f"litellm_proxy;model_id:model-123;llm_batch_id:batch_{name}"),
created_by="user-1",
created_at=now - age,
updated_at=now,
status="in_progress",
)
for name, age in (("old", timedelta(days=3)), ("new", timedelta(hours=1)))
)
prisma: Final = self._prisma([])
prisma.db.litellm_managedobjecttable = _FakeManagedObjectPages(rows)
llm_router: Final = MagicMock()
llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress"))
poller: Final = self._instance(prisma, llm_router)
async def _batches_polled_in_one_cycle() -> tuple[str, ...]:
llm_router.aretrieve_batch.reset_mock()
await poller.check_batch_cost()
return tuple(sorted(call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list))
assert [await _batches_polled_in_one_cycle() for _ in range(4)] == [("batch_new", "batch_old")] * 4
def _unbilled_rows(self, ages: dict[str, timedelta]) -> tuple[SimpleNamespace, ...]:
now: Final = datetime.now(timezone.utc)
return tuple(
SimpleNamespace(
id=f"job-{name}",
unified_object_id=self._encode(f"litellm_proxy;model_id:model-123;llm_batch_id:batch_{name}"),
created_by="user-1",
created_at=now - age,
updated_at=now,
status="in_progress",
)
for name, age in ages.items()
)
async def _poll_cycles(self, monkeypatch, ages: dict[str, timedelta], cycles: int, page_size: int = 2):
from litellm_enterprise.proxy.common_utils import check_batch_cost
monkeypatch.setattr(check_batch_cost, "MAX_OBJECTS_PER_POLL_CYCLE", page_size)
prisma: Final = self._prisma([])
prisma.db.litellm_managedobjecttable = _FakeManagedObjectPages(self._unbilled_rows(ages))
llm_router: Final = MagicMock()
llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress"))
poller: Final = self._instance(prisma, llm_router)
async def _batches_polled_in_one_cycle() -> frozenset[str]:
llm_router.aretrieve_batch.reset_mock()
await poller.check_batch_cost()
return frozenset(call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list)
return [await _batches_polled_in_one_cycle() for _ in range(cycles)]
@pytest.mark.asyncio
async def test_an_old_batch_is_polled_every_cycle_while_new_batches_fill_the_page(self, monkeypatch):
new_batches: Final = {f"new{n}": timedelta(hours=5 - n) for n in range(1, 5)}
polled: Final = await self._poll_cycles(
monkeypatch, {"old": timedelta(days=3), **new_batches}, cycles=8
)
assert all("batch_old" in cycle for cycle in polled)
assert all(f"batch_{name}" in frozenset().union(*polled[:4]) for name in new_batches)
@pytest.mark.asyncio
async def test_old_batches_that_fill_the_page_never_push_a_new_batch_out(self, monkeypatch):
polled: Final = await self._poll_cycles(
monkeypatch,
{"old1": timedelta(days=3, hours=1), "old2": timedelta(days=3), "new": timedelta(hours=1)},
cycles=6,
)
assert all("batch_new" in cycle for cycle in polled)
assert frozenset({"batch_old1", "batch_old2"}) <= frozenset().union(*polled[:2])
@pytest.mark.asyncio
async def test_old_and_new_batches_each_get_half_the_page_when_both_overflow_it(self, monkeypatch):
ages: Final = {
**{f"old{n}": timedelta(days=3, hours=n) for n in range(3)},
**{f"new{n}": timedelta(hours=n + 1) for n in range(3)},
}
polled: Final = await self._poll_cycles(monkeypatch, ages, cycles=6)
assert [sorted(batch_id[len("batch_") : -1] for batch_id in cycle) for cycle in polled] == [["new", "old"]] * 6
assert frozenset().union(*polled) == frozenset(f"batch_{name}" for name in ages)
@pytest.mark.asyncio
async def test_old_and_new_batches_take_turns_on_a_one_row_page(self, monkeypatch):
polled: Final = await self._poll_cycles(
monkeypatch, {"old": timedelta(days=3), "new": timedelta(hours=1)}, cycles=4, page_size=1
)
assert polled == [frozenset({"batch_old"}), frozenset({"batch_new"})] * 2
@pytest.mark.asyncio
async def test_new_batches_fill_the_page_when_no_old_batch_needs_polling(self, monkeypatch):
polled: Final = await self._poll_cycles(
monkeypatch, {f"new{n}": timedelta(hours=n + 1) for n in range(3)}, cycles=4
)
assert [len(cycle) for cycle in polled] == [2] * 4
@pytest.mark.asyncio
async def test_a_failed_expiry_write_leaves_the_rest_of_the_page_polled(self):
rows: Final = self._unbilled_rows(
{"old": timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, hours=1), "new": timedelta(hours=1)}
)
table: Final = _FakeManagedObjectPages(rows)
table.update_many = AsyncMock(side_effect=RuntimeError("database unavailable"))
prisma: Final = self._prisma([])
prisma.db.litellm_managedobjecttable = table
llm_router: Final = MagicMock()
llm_router.aretrieve_batch = AsyncMock(side_effect=_A_REJECTED_KEY)
poller: Final = self._instance(prisma, llm_router)
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY - 1):
await poller.check_batch_cost()
llm_router.aretrieve_batch.reset_mock()
await poller.check_batch_cost()
assert table.update_many.await_count == 1
assert {call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list} == {
"batch_old",
"batch_new",
}
class _FakeManagedObjectPages:
"""A LiteLLM_ManagedObjectTable double over several unbilled rows that pages them the way Postgres would:
inside the lane's created_at window, after the keyset cursor, in created_at then id order, take at most."""
def __init__(self, rows):
self.rows = rows
self.find_many = AsyncMock(side_effect=self._find_many)
self.find_first = AsyncMock(return_value=None)
self.update_many = AsyncMock(return_value=0)
self.update = AsyncMock()
@staticmethod
def _selected(row, where: dict) -> bool:
window: Final = where["created_at"]
cursor: Final = where.get("OR")
return (
row.created_at <= window["lte"]
and ("gt" not in window or row.created_at > window["gt"])
and (
cursor is None
or row.created_at > cursor[0]["created_at"]["gt"]
or (row.created_at == cursor[1]["created_at"] and row.id > cursor[1]["id"]["gt"])
)
)
async def _find_many(self, *, where: dict, take: int, order: object) -> tuple:
selected: Final = (row for row in self.rows if self._selected(row, where))
return tuple(sorted(selected, key=lambda row: (row.created_at, row.id))[:take])
class _FakeManagedObjectRow:
"""One managed batch row the provider has finished but nothing has costed yet."""
@ -2940,9 +3088,12 @@ class _FakeManagedObjectRow:
class _FakeManagedObjectTable:
"""A LiteLLM_ManagedObjectTable double backed by one real, mutable row.
It honours the batch_processed and status filters, so the poller's compare-and-swap
and the managed-files deletion guard both read the same state a shared Postgres row
would give them. Staleness sweeps (the only queries scoped by created_at) never match.
It honours the batch_processed and status filters and the deletion guard's created_at cutoff, so the
poller's compare-and-swap and the managed-files deletion guard both read the state a shared Postgres row
would give them. The poll lanes' created_at windows are ignored, so every lane sees the row, and each
read hands back a snapshot, so a write that lands mid-cycle is invisible to the row the poller holds.
Writes journal "claim" or "release" when they flip batch_processed, "expire" when they mark the row
stale_expired, and "finalize" otherwise.
"""
def __init__(self, row: _FakeManagedObjectRow, journal: list):
@ -2956,7 +3107,7 @@ class _FakeManagedObjectTable:
def _matches(self, where: dict) -> bool:
for key, value in where.items():
if key == "created_at":
if "lt" in value and not self.row.created_at < value["lt"]:
if "gte" in value and not self.row.created_at >= value["gte"]:
return False
continue
if key == "status":
@ -2971,10 +3122,12 @@ class _FakeManagedObjectTable:
async def _update_many(self, *, where: dict, data: dict) -> int:
if not self._matches(where):
return 0
if "updated_at" in where:
if "batch_processed" in data:
self.journal.append("claim" if data["batch_processed"] else "release")
elif data.get("status") == "stale_expired":
self.journal.append("expire")
else:
self.journal.append("finalize")
elif "batch_processed" in where:
self.journal.append("claim" if data.get("batch_processed") else "release")
for key, value in data.items():
setattr(self.row, key, value)
return 1
@ -2985,7 +3138,7 @@ class _FakeManagedObjectTable:
setattr(self.row, key, value)
async def _find_many(self, *, where: dict, take=None, order=None) -> list:
return [self.row] if self._matches(where) else []
return [copy(self.row)] if self._matches(where) else []
class TestMultiPodBatchCostClaim:
@ -3683,12 +3836,15 @@ class TestBatchLineItemsFromThePoller:
@classmethod
@contextmanager
def _provider(cls, journal: list):
"""The provider's batch files by id, and a spend-log writer that journals each write."""
def _provider(cls, journal: list, missing_file_error: Exception | None = None):
"""The provider's batch files by id, and a spend-log writer that journals each write. Reading any other
file raises missing_file_error, or the NotFoundError OpenAI answers with."""
files = {"file-input-1": cls._INPUT, _CLAIM_OUTPUT_FILE_ID: cls._OUTPUT, "file-err-1": cls._ERROR}
async def _afile_content(**kwargs):
journal.append(f"read {kwargs['file_id']}")
if kwargs["file_id"] not in files and missing_file_error is not None:
raise missing_file_error
if kwargs["file_id"] not in files:
from litellm.exceptions import NotFoundError
@ -3861,7 +4017,7 @@ class TestBatchLineItemsFromThePoller:
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
assert journal == ["claim", "read file-err-1", "read file-input-1", "lines"]
(payload,) = logger.payloads
assert [(line["custom_id"], line["status"], line["response_cost"]) for line in payload["lines"]] == [
("ok-1", "failure", 0.0)
@ -3892,7 +4048,7 @@ class TestBatchLineItemsFromThePoller:
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
assert journal == ["claim", "read file-err-1", "read file-input-1", "lines"]
((line,),) = [payload["lines"] for payload in logger.payloads]
assert line["request"] == json.loads(self._INPUT)
@ -3910,7 +4066,7 @@ class TestBatchLineItemsFromThePoller:
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize", "read file-err-1", "read file-input-1"]
assert journal == ["claim", "read file-err-1", "read file-input-1"]
assert logger.payloads == []
@pytest.mark.asyncio
@ -3970,11 +4126,37 @@ class TestBatchLineItemsFromThePoller:
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["read file-out-deleted", "finalize", "read file-err-1", "read file-input-1", "lines"]
assert journal == ["read file-out-deleted", "claim", "read file-err-1", "read file-input-1", "lines"]
(payload,) = logger.payloads
assert [(line["custom_id"], line["status"]) for line in payload["lines"]] == [("ok-1", "failure")]
assert row.batch_processed is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status_code", "missing_object", "retired"),
(
(404, "bucket/job-1/predictions.jsonl", True),
(404, "bucket/job-2/predictions.jsonl", False),
(500, "bucket/job-1/predictions.jsonl", False),
),
)
async def test_a_storage_404_for_the_output_object_retires_the_batch_unbilled(
self, status_code, missing_object, retired
):
"""Vertex AI reports a deleted gs:// output as a VertexAIError 404 that names the object without its
scheme, and no later poll can read that object either."""
from litellm.llms.vertex_ai.common_utils import VertexAIError
journal: list = []
row, instance = self._setup(journal, output_file_id="gs://bucket/job-1/predictions.jsonl")
missing: Final = VertexAIError(status_code=status_code, message=f"No such object: {missing_object}")
with self._provider(journal, missing_file_error=missing):
await instance.check_batch_cost()
assert row.batch_processed is retired
assert "bill" not in journal
@pytest.mark.asyncio
async def test_a_terminal_batch_with_no_lines_sends_nothing(self, monkeypatch):
import litellm
@ -3988,7 +4170,7 @@ class TestBatchLineItemsFromThePoller:
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize"]
assert journal == ["claim"]
assert logger.payloads == []
@pytest.mark.asyncio
@ -4088,16 +4270,19 @@ class TestBatchLineItemsFromThePoller:
_, instance = self._setup(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))
async def _an_earlier_batch_waits_to_store_its_lines():
find_jobs: Final = instance._find_jobs
async def _an_earlier_batch_waits_to_store_its_lines(where):
instance._store_batch_line_items_in_background(
MagicMock(), MagicMock(), "model-123", "batch-earlier", billed=True
)
await asyncio.sleep(0.01)
return await find_jobs(where)
async def _store(job, response, model_id, batch_id, *, billed):
journal.append(f"store {batch_id}")
monkeypatch.setattr(instance, "_cleanup_stale_managed_objects", _an_earlier_batch_waits_to_store_its_lines)
monkeypatch.setattr(instance, "_find_jobs", _an_earlier_batch_waits_to_store_its_lines)
monkeypatch.setattr(instance, "_store_batch_line_items", _store)
with self._provider(journal):
await instance.check_batch_cost()
@ -4345,23 +4530,317 @@ class TestCreatorTurnedOffMessageLogging:
instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called()
_PAST_THE_STALENESS_CUTOFF: Final = timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, hours=1)
_INSIDE_THE_STALENESS_CUTOFF: Final = timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - timedelta(hours=1)
def _provider_batch(status: str) -> LiteLLMBatch:
return LiteLLMBatch(
id="batch-456",
status=status,
output_file_id=None,
error_file_id=None,
created_at=1,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
)
@pytest.mark.asyncio
async def test_old_completed_batch_remains_available_for_costing() -> None:
async def test_an_unbilled_batch_past_the_staleness_cutoff_no_longer_blocks_deleting_its_input_file() -> None:
row: Final = _FakeManagedObjectRow()
row.status = "completed"
row.created_at = datetime(2000, 1, 1, tzinfo=timezone.utc)
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
await poller._cleanup_stale_managed_objects()
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, "file-input-1")
assert row.batch_processed is False
@pytest.mark.asyncio
@pytest.mark.parametrize(
("age", "file_field", "file_id"),
(
(_INSIDE_THE_STALENESS_CUTOFF, "input_file_id", "file-input-1"),
(_PAST_THE_STALENESS_CUTOFF, "output_file_id", _CLAIM_OUTPUT_FILE_ID),
(_PAST_THE_STALENESS_CUTOFF, "error_file_id", "file-error-1"),
),
ids=("input_inside_the_cutoff", "output_past_the_cutoff", "error_past_the_cutoff"),
)
async def test_an_unbilled_batch_still_blocks_deleting_a_file_its_cost_may_need(
age: timedelta, file_field: str, file_id: str
) -> None:
row: Final = _FakeManagedObjectRow()
row.status = "completed"
row.created_at = datetime.now(timezone.utc) - age
row.file_object = json.dumps({"id": "batch-456", "status": "completed", file_field: file_id})
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
with pytest.raises(HTTPException) as refused:
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, file_id)
assert refused.value.status_code == 400
@pytest.mark.asyncio
async def test_a_stale_expired_batch_no_longer_blocks_deleting_its_output_file() -> None:
row: Final = _FakeManagedObjectRow()
row.status = "stale_expired"
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
assert row.status == "stale_expired"
@pytest.mark.asyncio
async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_still_billed() -> None:
row: Final = _FakeManagedObjectRow()
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
journal: Final = []
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, journal)
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
with TestMultiPodBatchCostClaim._billing_patches(journal):
await poller.check_batch_cost()
assert journal == ["fetch", "claim", "bill"]
assert row.batch_processed is True
_A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", llm_provider="openai", model="gpt-4")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("age", "db_status", "unified_object_id", "retrieve", "expected_status"),
(
(
_PAST_THE_STALENESS_CUTOFF,
"in_progress",
_CLAIM_UNIFIED_BATCH_ID,
[_provider_batch("in_progress")] * UNSETTLED_POLLS_BEFORE_EXPIRY,
"in_progress",
),
(_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "in_progress"),
(
_PAST_THE_STALENESS_CUTOFF,
"in_progress",
_CLAIM_UNIFIED_BATCH_ID,
litellm.InternalServerError(message="upstream error", llm_provider="openai", model="gpt-4"),
"in_progress",
),
(
_PAST_THE_STALENESS_CUTOFF,
"in_progress",
_CLAIM_UNIFIED_BATCH_ID,
litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4"),
"in_progress",
),
(_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "stale_expired"),
(
_PAST_THE_STALENESS_CUTOFF,
"in_progress",
"batch-raw-1",
[_provider_batch("in_progress")] * UNSETTLED_POLLS_BEFORE_EXPIRY,
"stale_expired",
),
(_INSIDE_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "in_progress"),
(_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "completed"),
(_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "completed"),
),
ids=(
"still_running",
"retrieve_timed_out",
"provider_5xx",
"provider_429",
"key_rejected",
"unroutable",
"inside_the_cutoff",
"finished_and_retrieve_timed_out",
"finished_and_key_rejected",
),
)
async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_staleness_cutoff(
age: timedelta, db_status: str, unified_object_id: str, retrieve: object, expected_status: str
) -> None:
row: Final = _FakeManagedObjectRow()
row.created_at = datetime.now(timezone.utc) - age
row.status = db_status
row.unified_object_id = unified_object_id
router: Final = TestMultiPodBatchCostClaim._router()
router.aretrieve_batch.side_effect = retrieve
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router)
with TestMultiPodBatchCostClaim._billing_patches([]):
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
await poller.check_batch_cost()
assert row.status == expected_status
assert row.batch_processed is False
@pytest.mark.asyncio
async def test_an_unfinished_batch_past_the_staleness_cutoff_expires_only_after_consecutive_refused_retrieves() -> None:
row: Final = _FakeManagedObjectRow()
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
router: Final = TestMultiPodBatchCostClaim._router()
router.aretrieve_batch.side_effect = [
*([_A_REJECTED_KEY] * (UNSETTLED_POLLS_BEFORE_EXPIRY - 1)),
_provider_batch("in_progress"),
*([_A_REJECTED_KEY] * UNSETTLED_POLLS_BEFORE_EXPIRY),
]
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router)
statuses: Final = []
with TestMultiPodBatchCostClaim._billing_patches([]):
for _ in range(2 * UNSETTLED_POLLS_BEFORE_EXPIRY):
await poller.check_batch_cost()
statuses.append(row.status)
assert statuses == [*(["in_progress"] * (2 * UNSETTLED_POLLS_BEFORE_EXPIRY - 1)), "stale_expired"]
async def _a_vertex_job_that_no_longer_exists(*, model: str, batch_id: str, litellm_metadata: object) -> LiteLLMBatch:
from litellm.llms.vertex_ai.common_utils import VertexAIError
raise VertexAIError(
status_code=404,
message='Error: 404 {"error":{"code":404,"message":"The BatchPredictionJob does not exist.","status":"NOT_FOUND"}}',
)
async def _a_finished_batch_whose_output_never_appeared(
*, model: str, batch_id: str, litellm_metadata: object
) -> LiteLLMBatch:
return _provider_batch("completed")
async def _an_output_file_the_key_cannot_read() -> None:
raise litellm.PermissionDeniedError(
message="forbidden",
llm_provider="openai",
model="gpt-4",
response=httpx.Response(403, request=httpx.Request("GET", "https://api.openai.com/v1/files")),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(("age", "retired"), ((_PAST_THE_STALENESS_CUTOFF, True), (_INSIDE_THE_STALENESS_CUTOFF, False)))
async def test_a_finished_batch_whose_output_never_appeared_frees_its_files_once_past_the_staleness_cutoff(
age: timedelta, retired: bool
) -> None:
row: Final = _FakeManagedObjectRow()
row.created_at = datetime.now(timezone.utc) - age
row.status = "completed"
router: Final = TestMultiPodBatchCostClaim._router()
router.aretrieve_batch.side_effect = _a_finished_batch_whose_output_never_appeared
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router)
with TestMultiPodBatchCostClaim._billing_patches([]):
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
await poller.check_batch_cost()
assert (row.status, row.batch_processed) == ("completed", retired)
if retired:
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
else:
with pytest.raises(HTTPException):
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
@pytest.mark.asyncio
@pytest.mark.parametrize("db_status", ("in_progress", "completed"))
async def test_a_batch_past_the_staleness_cutoff_whose_output_id_lags_is_billed_once_it_appears(db_status: str) -> None:
row: Final = _FakeManagedObjectRow()
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
row.status = db_status
journal: Final = []
router: Final = TestMultiPodBatchCostClaim._router()
router.aretrieve_batch.side_effect = [
*([_provider_batch("completed")] * (UNSETTLED_POLLS_BEFORE_EXPIRY - 1)),
router.aretrieve_batch.return_value,
]
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, journal), router)
with TestMultiPodBatchCostClaim._billing_patches(journal):
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
await poller.check_batch_cost()
assert journal == ["fetch", "claim", "bill"]
assert row.batch_processed is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
("retrieve", "during_fetch"),
(
(_A_REJECTED_KEY, None),
(_a_vertex_job_that_no_longer_exists, None),
(None, _an_output_file_the_key_cannot_read),
),
ids=("key_rejected", "batch_404_without_its_id", "output_unreadable"),
)
async def test_a_finished_batch_the_provider_refuses_past_the_staleness_cutoff_bills_once_the_refusal_is_fixed(
retrieve: object, during_fetch: object
) -> None:
row: Final = _FakeManagedObjectRow()
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
row.status = "completed"
router: Final = TestMultiPodBatchCostClaim._router()
router.aretrieve_batch.side_effect = retrieve
journal: Final = []
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, journal)
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router)
with TestMultiPodBatchCostClaim._billing_patches(journal, during_fetch=during_fetch):
await poller.check_batch_cost()
assert (row.status, row.batch_processed) == ("completed", False)
with pytest.raises(HTTPException):
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
router.aretrieve_batch.side_effect = None
with TestMultiPodBatchCostClaim._billing_patches(journal):
await poller.check_batch_cost()
assert journal[-1] == "bill"
assert row.batch_processed is True
@pytest.mark.asyncio
async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_status_a_client_wrote_meanwhile() -> None:
row: Final = _FakeManagedObjectRow()
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
router: Final = TestMultiPodBatchCostClaim._router()
async def _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_is_refused(
*, model: str, batch_id: str, litellm_metadata: object
) -> LiteLLMBatch:
if router.aretrieve_batch.call_count == UNSETTLED_POLLS_BEFORE_EXPIRY:
row.status = "completed"
raise _A_REJECTED_KEY
router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_is_refused
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router)
with TestMultiPodBatchCostClaim._billing_patches([]):
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
await poller.check_batch_cost()
assert row.status == "completed"
assert row.batch_processed is False
@pytest.mark.asyncio
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
async def test_stale_retirement_preserves_concurrent_progress(progress: str) -> None:
async def test_stale_retirement_lands_unless_another_pod_claimed_the_row(progress: str) -> None:
"""A client GET that rewrote the row while the poller held it moves updated_at but leaves the batch unbilled,
so the retirement still lands."""
row: Final = _FakeManagedObjectRow()
selected: Final = copy(row)
if progress == "claimed":
@ -4372,13 +4851,16 @@ async def test_stale_retirement_preserves_concurrent_progress(progress: str) ->
await poller._retire_job(selected, "provider batch disappeared")
assert row.batch_processed is (progress == "claimed")
assert row.updated_at > selected.updated_at
assert row.batch_processed is True
@pytest.mark.asyncio
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
async def test_stale_terminal_response_preserves_concurrent_progress(progress: str) -> None:
@pytest.mark.parametrize(
("progress", "finalized", "expected_status"), [("claimed", False, "in_progress"), ("refreshed", True, "failed")]
)
async def test_stale_terminal_response_lands_unless_another_pod_claimed_the_row(
progress: str, finalized: bool, expected_status: str
) -> None:
from litellm.types.utils import LiteLLMBatch
row: Final = _FakeManagedObjectRow()
@ -4399,11 +4881,10 @@ async def test_stale_terminal_response_preserves_concurrent_progress(progress: s
status="failed",
)
assert await poller._finalize_unbilled_terminal_job(selected, response) is False
assert await poller._finalize_unbilled_terminal_job(selected, response) is finalized
assert row.file_object == '{"revision":"fresh"}'
assert row.status == "in_progress"
assert row.batch_processed is (progress == "claimed")
assert row.status == expected_status
assert row.batch_processed is True
@pytest.mark.asyncio
@ -4457,6 +4938,23 @@ def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, mes
assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True
@pytest.mark.parametrize(
("missing_key", "gone"), (("out/job-123/input.jsonl.out", True), ("out/job-123/other.jsonl.out", False))
)
def test_an_s3_no_such_key_for_the_output_object_is_permanent_loss(missing_key: str, gone: bool) -> None:
from litellm.llms.bedrock.common_utils import BedrockError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
# S3 GetObject 404 body shape, checked live against a missing key on 2026-10-20
error: Final = BedrockError(
status_code=404,
message='<?xml version="1.0" encoding="UTF-8"?>\n<Error><Code>NoSuchKey</Code>'
f"<Message>The specified key does not exist.</Message><Key>{missing_key}</Key>"
"<RequestId>R</RequestId><HostId>H</HostId></Error>",
)
assert CheckBatchCost._is_output_file_gone_at_provider(error, "s3://qa-bucket/out/job-123/input.jsonl.out") is gone
@pytest.mark.parametrize("output_file_id", (None, ""))
def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None:
from litellm.exceptions import NotFoundError