mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 33e2700632 into 969161501a
This commit is contained in:
commit
a1a45a5791
4 changed files with 702 additions and 122 deletions
|
|
@ -42,11 +42,12 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
def _error_names_resource(error: Exception, resource_id: str) -> bool:
|
||||
return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
|
||||
return resource_id in re.findall(r"""[^\s"'`<>]+""", str(error))
|
||||
|
||||
|
||||
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
|
||||
RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
|
||||
UNSETTLED_POLLS_BEFORE_EXPIRY: Final = 3
|
||||
|
||||
PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
|
||||
"completed",
|
||||
|
|
@ -91,6 +92,10 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
|
|||
return TeamRepository(prisma_client).table
|
||||
|
||||
|
||||
def _unique_jobs(jobs: Sequence[ManagedObjectRow]) -> tuple[ManagedObjectRow, ...]:
|
||||
return tuple({job.id: job for job in jobs}.values())
|
||||
|
||||
|
||||
class CheckBatchCost:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -116,6 +121,7 @@ class CheckBatchCost:
|
|||
self._arrival_cursor: tuple[datetime, str] | None = None
|
||||
self._arrival_before: datetime = self._poll_before
|
||||
self._poll_arrivals: bool = False
|
||||
self._unsettled_polls: Mapping[str, int] = MappingProxyType({})
|
||||
self._line_item_stores: Final[set[asyncio.Task[None]]] = set()
|
||||
self._line_item_store_turn: Final = asyncio.Lock()
|
||||
self._poll_idle: Final = asyncio.Event()
|
||||
|
|
@ -147,27 +153,6 @@ class CheckBatchCost:
|
|||
return
|
||||
self.batch_processed_support_confirmed = True
|
||||
|
||||
async def _cleanup_stale_managed_objects(self) -> None:
|
||||
"""
|
||||
Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
|
||||
in non-terminal states as 'stale_expired'. These will never complete and
|
||||
should not be polled.
|
||||
"""
|
||||
cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
|
||||
result: Final = await _managed_object_table(self.prisma_client).update_many(
|
||||
where={
|
||||
"file_purpose": "batch",
|
||||
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
|
||||
"created_at": {"lt": cutoff},
|
||||
},
|
||||
data={"status": "stale_expired"},
|
||||
)
|
||||
if result > 0:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: marked {result} stale managed objects "
|
||||
f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
|
||||
)
|
||||
|
||||
async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]":
|
||||
"""Query batch jobs without the batch_processed filter (for older schemas)."""
|
||||
return await self._find_jobs(
|
||||
|
|
@ -187,14 +172,37 @@ class CheckBatchCost:
|
|||
)
|
||||
|
||||
async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]:
|
||||
arrivals: Final = self._poll_arrivals
|
||||
page: Final = await self._find_job_page(where, arrivals=arrivals)
|
||||
self._poll_arrivals = not arrivals if page else arrivals
|
||||
if page:
|
||||
return page
|
||||
return await self._find_job_page(where, arrivals=not arrivals)
|
||||
lead_arrivals: Final = self._poll_arrivals
|
||||
self._poll_arrivals = not lead_arrivals
|
||||
lead_share: Final = MAX_OBJECTS_PER_POLL_CYCLE - MAX_OBJECTS_PER_POLL_CYCLE // 2
|
||||
lead: Final = await self._find_job_page(where, arrivals=lead_arrivals, take=lead_share)
|
||||
trail: Final = await self._find_job_page(
|
||||
where, arrivals=not lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead)
|
||||
)
|
||||
spill: Final = (
|
||||
await self._find_job_page(
|
||||
where, arrivals=lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead) - len(trail)
|
||||
)
|
||||
if len(lead) == lead_share
|
||||
else ()
|
||||
)
|
||||
return _unique_jobs((*lead, *trail, *spill))
|
||||
|
||||
async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]:
|
||||
async def _find_job_page(
|
||||
self, where: Mapping[str, object], *, arrivals: bool, take: int
|
||||
) -> Sequence[ManagedObjectRow]:
|
||||
if take == 0:
|
||||
return ()
|
||||
resumed_mid_lane: Final = (self._arrival_cursor if arrivals else self._poll_cursor) is not None
|
||||
page: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take)
|
||||
if len(page) == take or not resumed_mid_lane:
|
||||
return page
|
||||
wrapped: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take - len(page))
|
||||
return _unique_jobs((*page, *wrapped))
|
||||
|
||||
async def _find_job_page_from_cursor(
|
||||
self, where: Mapping[str, object], *, arrivals: bool, take: int
|
||||
) -> Sequence[ManagedObjectRow]:
|
||||
cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
|
||||
if cursor is None:
|
||||
if arrivals:
|
||||
|
|
@ -220,10 +228,10 @@ class CheckBatchCost:
|
|||
else {}
|
||||
),
|
||||
},
|
||||
take=MAX_OBJECTS_PER_POLL_CYCLE,
|
||||
take=take,
|
||||
order=[{"created_at": "asc"}, {"id": "asc"}],
|
||||
)
|
||||
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
|
||||
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == take else None
|
||||
if arrivals:
|
||||
self._arrival_cursor = next_cursor
|
||||
else:
|
||||
|
|
@ -231,11 +239,65 @@ class CheckBatchCost:
|
|||
return page
|
||||
|
||||
def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]:
|
||||
return {
|
||||
"id": job.id,
|
||||
"updated_at": job.updated_at,
|
||||
**({"batch_processed": False} if self._has_batch_processed_column else {}),
|
||||
}
|
||||
if self._has_batch_processed_column:
|
||||
return {"id": job.id, "batch_processed": False}
|
||||
return {"id": job.id, "updated_at": job.updated_at}
|
||||
|
||||
@staticmethod
|
||||
def _is_past_staleness_cutoff(job: ManagedObjectRow) -> bool:
|
||||
return job.created_at < datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
|
||||
|
||||
async def _expire_if_stale(self, job: ManagedObjectRow) -> None:
|
||||
if not self._is_past_staleness_cutoff(job):
|
||||
return
|
||||
if job.status in TERMINAL_MANAGED_OBJECT_STATUSES:
|
||||
await self._retire_job(job, f"unbilled {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days after it was created")
|
||||
return
|
||||
await self._expire_unfinished(job)
|
||||
|
||||
async def _expire_after_repeated_unsettled_polls(self, job: ManagedObjectRow) -> None:
|
||||
unsettled: Final = self._unsettled_polls.get(job.id, 0) + 1
|
||||
if unsettled < UNSETTLED_POLLS_BEFORE_EXPIRY:
|
||||
self._unsettled_polls = MappingProxyType({**self._unsettled_polls, job.id: unsettled})
|
||||
return
|
||||
self._forget_unsettled_polls(job)
|
||||
await self._expire_if_stale(job)
|
||||
|
||||
def _forget_unsettled_polls(self, job: ManagedObjectRow) -> None:
|
||||
if job.id in self._unsettled_polls:
|
||||
self._unsettled_polls = MappingProxyType(
|
||||
{job_id: count for job_id, count in self._unsettled_polls.items() if job_id != job.id}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_completed_with_output_id_pending(response: "LiteLLMBatch") -> bool:
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
_completed_batch_safe_to_retire,
|
||||
)
|
||||
|
||||
return (
|
||||
response.status in ("completed", "complete")
|
||||
and response.output_file_id is None
|
||||
and not _completed_batch_safe_to_retire(response)
|
||||
)
|
||||
|
||||
async def _expire_unfinished(self, job: ManagedObjectRow) -> None:
|
||||
try:
|
||||
expired: Final = await _managed_object_table(self.prisma_client).update_many(
|
||||
where={
|
||||
**self._unprocessed_snapshot(job),
|
||||
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
|
||||
},
|
||||
data={"status": "stale_expired"},
|
||||
)
|
||||
except Exception as db_err:
|
||||
verbose_proxy_logger.error(f"CheckBatchCost: failed to mark stale job {job.id} as stale_expired: {db_err}")
|
||||
return
|
||||
if expired > 0:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: job {job.id} is older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days "
|
||||
"and still not finished, so it is marked stale_expired and no longer polled"
|
||||
)
|
||||
|
||||
async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None:
|
||||
"""
|
||||
|
|
@ -345,6 +407,16 @@ class CheckBatchCost:
|
|||
|
||||
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id)
|
||||
|
||||
@staticmethod
|
||||
def _is_lasting_provider_failure(error: Exception) -> bool:
|
||||
import openai
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
return isinstance(error, (BaseLLMException, openai.APIStatusError)) and not litellm.should_retry(
|
||||
error.status_code
|
||||
)
|
||||
|
||||
def _batch_deployment_exists(self, model_id: str) -> bool:
|
||||
"""A 404 only proves the batch is gone when it came from the batch's own
|
||||
deployment. Once that deployment leaves the router, default fallbacks can
|
||||
|
|
@ -356,15 +428,21 @@ class CheckBatchCost:
|
|||
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
|
||||
"""A 404 naming the output file means there is nothing to fetch on this or any
|
||||
later poll: providers like Vertex AI advertise an output path for every batch,
|
||||
including terminal ones that never wrote it. Any other failure may be
|
||||
transient, so it keeps retrying until the staleness sweep bounds it."""
|
||||
including terminal ones that never wrote it. GCS names a gs:// object without its scheme,
|
||||
and S3 names an s3:// object by its key alone. Any other failure keeps retrying."""
|
||||
import openai
|
||||
|
||||
from litellm.exceptions import NotFoundError
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
if not output_file_id:
|
||||
return False
|
||||
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
|
||||
is_not_found: Final = isinstance(error, (NotFoundError, openai.NotFoundError)) or (
|
||||
isinstance(error, BaseLLMException) and error.status_code == 404
|
||||
)
|
||||
object_path: Final = output_file_id.split("://", 1)[-1]
|
||||
names: Final = (output_file_id, object_path, object_path.split("/", 1)[-1])
|
||||
return is_not_found and any(_error_names_resource(error, name) for name in names)
|
||||
|
||||
async def _finalize_unbilled_terminal_job(
|
||||
self, job: "ManagedObjectRow", response: "LiteLLMBatch"
|
||||
|
|
@ -1094,13 +1172,6 @@ class CheckBatchCost:
|
|||
|
||||
processed_models: List[Tuple[Optional[str], Optional[str]]] = []
|
||||
|
||||
try:
|
||||
await self._cleanup_stale_managed_objects()
|
||||
except Exception as cleanup_err:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: stale cleanup failed (poll will continue): {cleanup_err}"
|
||||
)
|
||||
|
||||
# Look for all batches that have not yet been processed by CheckBatchCost.
|
||||
# self._has_batch_processed_column is cached after the first probe so that
|
||||
# older schemas don't pay a guaranteed-failing primary query + warning on
|
||||
|
|
@ -1142,6 +1213,8 @@ class CheckBatchCost:
|
|||
if routing is None:
|
||||
if self._has_unified_id_without_model(job):
|
||||
await self._retire_job(job, "unified object id has no model id")
|
||||
else:
|
||||
await self._expire_if_stale(job)
|
||||
continue
|
||||
model_id, batch_id = routing
|
||||
|
||||
|
|
@ -1166,7 +1239,22 @@ class CheckBatchCost:
|
|||
prom_logger.record_check_batch_cost_error("provider_retrieval_error")
|
||||
if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id):
|
||||
await self._retire_job(job, f"batch {batch_id} no longer exists at the provider")
|
||||
elif (
|
||||
self._is_past_staleness_cutoff(job)
|
||||
and job.status not in TERMINAL_MANAGED_OBJECT_STATUSES
|
||||
and self._is_lasting_provider_failure(e)
|
||||
):
|
||||
await self._expire_after_repeated_unsettled_polls(job)
|
||||
continue
|
||||
if self._is_completed_with_output_id_pending(response):
|
||||
verbose_proxy_logger.info(
|
||||
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
|
||||
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
|
||||
)
|
||||
if self._is_past_staleness_cutoff(job):
|
||||
await self._expire_after_repeated_unsettled_polls(job)
|
||||
continue
|
||||
self._forget_unsettled_polls(job)
|
||||
|
||||
## RETRIEVE THE BATCH JOB OUTPUT FILE
|
||||
if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
|
||||
|
|
@ -1217,16 +1305,6 @@ class CheckBatchCost:
|
|||
self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
|
||||
|
||||
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
_completed_batch_safe_to_retire,
|
||||
)
|
||||
|
||||
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
|
||||
verbose_proxy_logger.info(
|
||||
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
|
||||
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
|
||||
)
|
||||
continue
|
||||
await self._retire_unbilled_terminal_job(job, response, model_id, batch_id)
|
||||
|
||||
# Record polling run metrics (always, even if nothing was processed)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import base64
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
|
|
@ -39,6 +40,7 @@ from litellm.constants import (
|
|||
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS,
|
||||
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
|
||||
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
|
||||
MAX_FILE_LIST_LIMIT,
|
||||
PENDING_RESPONSE_STATUSES,
|
||||
)
|
||||
|
|
@ -2387,12 +2389,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
where={
|
||||
"file_purpose": "batch",
|
||||
"batch_processed": False,
|
||||
"status": {"not_in": ["failed", "expired", "cancelled"]},
|
||||
"status": {"not_in": ["failed", "expired", "cancelled", "stale_expired"]},
|
||||
},
|
||||
take=MAX_MATCHES_TO_RETURN,
|
||||
order={"created_at": "desc"},
|
||||
)
|
||||
|
||||
input_file_cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
|
||||
referencing_batches: Final[list[dict[str, object]]] = []
|
||||
for batch in batches:
|
||||
try:
|
||||
|
|
@ -2405,7 +2408,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# Extract file IDs from batch
|
||||
# Batches typically reference the unified file ID in input_file_id
|
||||
# Output and error files are generated by the provider
|
||||
input_file_id = batch_data.get("input_file_id")
|
||||
input_file_id = batch_data.get("input_file_id") if batch.created_at >= input_file_cutoff else None
|
||||
output_file_id = batch_data.get("output_file_id")
|
||||
error_file_id = batch_data.get("error_file_id")
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ This ensures cost tracking is not disrupted by premature file deletion.
|
|||
|
||||
import base64
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -50,7 +51,7 @@ def _make_batch_db_record(
|
|||
mock_batch.status = status
|
||||
mock_batch.file_object = json.dumps(file_object)
|
||||
mock_batch.created_by = created_by
|
||||
mock_batch.created_at = 1700000000
|
||||
mock_batch.created_at = datetime.now(timezone.utc)
|
||||
return mock_batch
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,19 +12,23 @@ import logging
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from copy import copy
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from hashlib import sha256
|
||||
from queue import SimpleQueue
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
from litellm_enterprise.proxy.common_utils.check_batch_cost import UNSETTLED_POLLS_BEFORE_EXPIRY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.batches.batch_utils import BatchCostUsageResult
|
||||
|
|
@ -154,23 +158,6 @@ class TestCheckBatchCost:
|
|||
llm_router=mock_llm_router,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleanup_scoped_to_batch_file_purpose(self, check_batch_cost_instance, mock_prisma_client):
|
||||
"""_cleanup_stale_managed_objects scopes its update to file_purpose='batch' only."""
|
||||
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
|
||||
# Return empty so the main poll loop exits immediately
|
||||
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])
|
||||
|
||||
await check_batch_cost_instance.check_batch_cost()
|
||||
|
||||
calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list
|
||||
stale_call = calls[0]
|
||||
assert stale_call[1]["data"] == {"status": "stale_expired"}
|
||||
where = stale_call[1]["where"]
|
||||
assert where["file_purpose"] == "batch"
|
||||
assert "stale_expired" in where["status"]["not_in"]
|
||||
assert "created_at" in where
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_probe_confirms_batch_processed_support(self, check_batch_cost_instance, mock_prisma_client):
|
||||
mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None)
|
||||
|
|
@ -251,7 +238,7 @@ class TestCheckBatchCost:
|
|||
fallback_where = calls[1][1]["where"]
|
||||
assert "batch_processed" not in fallback_where
|
||||
assert "stale_expired" in fallback_where["status"]["not_in"]
|
||||
assert calls[1][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
|
||||
assert calls[2][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
|
||||
# Column absence is now cached — next call should go straight to fallback
|
||||
assert check_batch_cost_instance._has_batch_processed_column is False
|
||||
assert check_batch_cost_instance.batch_processed_support_confirmed is False
|
||||
|
|
@ -744,7 +731,7 @@ class TestCheckBatchCost:
|
|||
|
||||
await check_batch_cost_instance.check_batch_cost()
|
||||
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
|
||||
"Expected update() to be called exactly once for the completed job"
|
||||
)
|
||||
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
|
||||
|
|
@ -877,7 +864,7 @@ class TestCheckBatchCost:
|
|||
|
||||
mock_update_database.assert_awaited_once()
|
||||
assert mock_update_database.call_args.kwargs["response_cost"] == 0.01
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
|
||||
"the job must still be marked processed once cost tracking succeeds"
|
||||
)
|
||||
|
||||
|
|
@ -979,7 +966,7 @@ class TestCheckBatchCost:
|
|||
|
||||
await check_batch_cost_instance.check_batch_cost()
|
||||
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
|
||||
f"Expected update() to be called exactly once for a {terminal_status} job"
|
||||
)
|
||||
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
|
||||
|
|
@ -1085,13 +1072,9 @@ class TestCheckBatchCost:
|
|||
assert owner.user_id == "user-1"
|
||||
assert owner.team_id == "team-1"
|
||||
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1
|
||||
update_call = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args
|
||||
assert update_call.kwargs["where"] == {
|
||||
"id": "job-terminal-mint-1",
|
||||
"batch_processed": False,
|
||||
"updated_at": mock_job.updated_at,
|
||||
}
|
||||
assert update_call.kwargs["where"] == {"id": "job-terminal-mint-1", "batch_processed": False}
|
||||
update_data = update_call.kwargs["data"]
|
||||
assert update_data["status"] == terminal_status
|
||||
assert update_data["batch_processed"] is True
|
||||
|
|
@ -1156,7 +1139,7 @@ class TestCheckBatchCost:
|
|||
) as mock_afile_content:
|
||||
await check_batch_cost_instance.check_batch_cost()
|
||||
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
|
||||
"a completed batch with no output file must be marked processed exactly once"
|
||||
)
|
||||
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
|
||||
|
|
@ -1205,6 +1188,7 @@ class TestCheckBatchCost:
|
|||
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
|
||||
).decode()
|
||||
mock_job.created_by = "user-1"
|
||||
mock_job.created_at = datetime.now(timezone.utc)
|
||||
|
||||
assert check_batch_cost_instance._has_batch_processed_column is True
|
||||
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
|
||||
|
|
@ -1245,6 +1229,7 @@ class TestCheckBatchCost:
|
|||
mock_job.id = "job-in-progress-1"
|
||||
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
|
||||
mock_job.created_by = "user-1"
|
||||
mock_job.created_at = datetime.now(timezone.utc)
|
||||
|
||||
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
|
||||
|
||||
|
|
@ -1375,7 +1360,7 @@ class TestCheckBatchCost:
|
|||
f"{terminal_status} batch with an output file must fetch results and be billed"
|
||||
)
|
||||
mock_logging_obj.async_success_handler.assert_awaited_once()
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1
|
||||
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
|
||||
assert update_data["batch_processed"] is True
|
||||
assert update_data["status"] == terminal_status, (
|
||||
|
|
@ -1548,7 +1533,7 @@ class TestCheckBatchCost:
|
|||
|
||||
assert mock_afile_content.await_count == 1
|
||||
mock_calculate.assert_not_awaited()
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
|
||||
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, (
|
||||
"a terminal batch with a 404ing output file must be retired, not retried forever"
|
||||
)
|
||||
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
|
||||
|
|
@ -1946,7 +1931,7 @@ class TestUnmanagedVertexRouting:
|
|||
mock_logging_obj.async_success_handler.assert_awaited_once()
|
||||
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01
|
||||
|
||||
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
|
||||
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 1
|
||||
update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
|
||||
assert update_data["batch_processed"] is True
|
||||
assert update_data["status"] == "complete"
|
||||
|
|
@ -2163,7 +2148,7 @@ class TestUnmanagedBedrockRouting:
|
|||
mock_logging_obj.async_success_handler.assert_awaited_once()
|
||||
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02
|
||||
|
||||
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
|
||||
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 1
|
||||
update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
|
||||
assert update_data["batch_processed"] is True
|
||||
assert update_data["status"] == "complete"
|
||||
|
|
@ -2725,6 +2710,7 @@ class TestPollPageStarvation:
|
|||
job.id = job_id
|
||||
job.unified_object_id = unified_object_id
|
||||
job.created_by = "user-1"
|
||||
job.created_at = datetime.now(timezone.utc)
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -2910,6 +2896,168 @@ class TestPollPageStarvation:
|
|||
|
||||
prisma.db.litellm_managedobjecttable.update.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_new_batch_is_polled_every_cycle_while_an_older_batch_stays_unbilled(self):
|
||||
"""An unbilled batch older than the arrival window used to take every other cycle away from the arrival
|
||||
lane, so a new batch waited two poll intervals to be billed."""
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
rows: Final = tuple(
|
||||
SimpleNamespace(
|
||||
id=f"job-{name}",
|
||||
unified_object_id=self._encode(f"litellm_proxy;model_id:model-123;llm_batch_id:batch_{name}"),
|
||||
created_by="user-1",
|
||||
created_at=now - age,
|
||||
updated_at=now,
|
||||
status="in_progress",
|
||||
)
|
||||
for name, age in (("old", timedelta(days=3)), ("new", timedelta(hours=1)))
|
||||
)
|
||||
prisma: Final = self._prisma([])
|
||||
prisma.db.litellm_managedobjecttable = _FakeManagedObjectPages(rows)
|
||||
llm_router: Final = MagicMock()
|
||||
llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress"))
|
||||
poller: Final = self._instance(prisma, llm_router)
|
||||
|
||||
async def _batches_polled_in_one_cycle() -> tuple[str, ...]:
|
||||
llm_router.aretrieve_batch.reset_mock()
|
||||
await poller.check_batch_cost()
|
||||
return tuple(sorted(call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list))
|
||||
|
||||
assert [await _batches_polled_in_one_cycle() for _ in range(4)] == [("batch_new", "batch_old")] * 4
|
||||
|
||||
def _unbilled_rows(self, ages: dict[str, timedelta]) -> tuple[SimpleNamespace, ...]:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
return tuple(
|
||||
SimpleNamespace(
|
||||
id=f"job-{name}",
|
||||
unified_object_id=self._encode(f"litellm_proxy;model_id:model-123;llm_batch_id:batch_{name}"),
|
||||
created_by="user-1",
|
||||
created_at=now - age,
|
||||
updated_at=now,
|
||||
status="in_progress",
|
||||
)
|
||||
for name, age in ages.items()
|
||||
)
|
||||
|
||||
async def _poll_cycles(self, monkeypatch, ages: dict[str, timedelta], cycles: int, page_size: int = 2):
|
||||
from litellm_enterprise.proxy.common_utils import check_batch_cost
|
||||
|
||||
monkeypatch.setattr(check_batch_cost, "MAX_OBJECTS_PER_POLL_CYCLE", page_size)
|
||||
prisma: Final = self._prisma([])
|
||||
prisma.db.litellm_managedobjecttable = _FakeManagedObjectPages(self._unbilled_rows(ages))
|
||||
llm_router: Final = MagicMock()
|
||||
llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress"))
|
||||
poller: Final = self._instance(prisma, llm_router)
|
||||
|
||||
async def _batches_polled_in_one_cycle() -> frozenset[str]:
|
||||
llm_router.aretrieve_batch.reset_mock()
|
||||
await poller.check_batch_cost()
|
||||
return frozenset(call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list)
|
||||
|
||||
return [await _batches_polled_in_one_cycle() for _ in range(cycles)]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_old_batch_is_polled_every_cycle_while_new_batches_fill_the_page(self, monkeypatch):
|
||||
new_batches: Final = {f"new{n}": timedelta(hours=5 - n) for n in range(1, 5)}
|
||||
polled: Final = await self._poll_cycles(
|
||||
monkeypatch, {"old": timedelta(days=3), **new_batches}, cycles=8
|
||||
)
|
||||
|
||||
assert all("batch_old" in cycle for cycle in polled)
|
||||
assert all(f"batch_{name}" in frozenset().union(*polled[:4]) for name in new_batches)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_batches_that_fill_the_page_never_push_a_new_batch_out(self, monkeypatch):
|
||||
polled: Final = await self._poll_cycles(
|
||||
monkeypatch,
|
||||
{"old1": timedelta(days=3, hours=1), "old2": timedelta(days=3), "new": timedelta(hours=1)},
|
||||
cycles=6,
|
||||
)
|
||||
|
||||
assert all("batch_new" in cycle for cycle in polled)
|
||||
assert frozenset({"batch_old1", "batch_old2"}) <= frozenset().union(*polled[:2])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_and_new_batches_each_get_half_the_page_when_both_overflow_it(self, monkeypatch):
|
||||
ages: Final = {
|
||||
**{f"old{n}": timedelta(days=3, hours=n) for n in range(3)},
|
||||
**{f"new{n}": timedelta(hours=n + 1) for n in range(3)},
|
||||
}
|
||||
polled: Final = await self._poll_cycles(monkeypatch, ages, cycles=6)
|
||||
|
||||
assert [sorted(batch_id[len("batch_") : -1] for batch_id in cycle) for cycle in polled] == [["new", "old"]] * 6
|
||||
assert frozenset().union(*polled) == frozenset(f"batch_{name}" for name in ages)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_and_new_batches_take_turns_on_a_one_row_page(self, monkeypatch):
|
||||
polled: Final = await self._poll_cycles(
|
||||
monkeypatch, {"old": timedelta(days=3), "new": timedelta(hours=1)}, cycles=4, page_size=1
|
||||
)
|
||||
|
||||
assert polled == [frozenset({"batch_old"}), frozenset({"batch_new"})] * 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_batches_fill_the_page_when_no_old_batch_needs_polling(self, monkeypatch):
|
||||
polled: Final = await self._poll_cycles(
|
||||
monkeypatch, {f"new{n}": timedelta(hours=n + 1) for n in range(3)}, cycles=4
|
||||
)
|
||||
|
||||
assert [len(cycle) for cycle in polled] == [2] * 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failed_expiry_write_leaves_the_rest_of_the_page_polled(self):
|
||||
rows: Final = self._unbilled_rows(
|
||||
{"old": timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, hours=1), "new": timedelta(hours=1)}
|
||||
)
|
||||
table: Final = _FakeManagedObjectPages(rows)
|
||||
table.update_many = AsyncMock(side_effect=RuntimeError("database unavailable"))
|
||||
prisma: Final = self._prisma([])
|
||||
prisma.db.litellm_managedobjecttable = table
|
||||
llm_router: Final = MagicMock()
|
||||
llm_router.aretrieve_batch = AsyncMock(side_effect=_A_REJECTED_KEY)
|
||||
poller: Final = self._instance(prisma, llm_router)
|
||||
|
||||
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY - 1):
|
||||
await poller.check_batch_cost()
|
||||
llm_router.aretrieve_batch.reset_mock()
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert table.update_many.await_count == 1
|
||||
assert {call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list} == {
|
||||
"batch_old",
|
||||
"batch_new",
|
||||
}
|
||||
|
||||
|
||||
class _FakeManagedObjectPages:
|
||||
"""A LiteLLM_ManagedObjectTable double over several unbilled rows that pages them the way Postgres would:
|
||||
inside the lane's created_at window, after the keyset cursor, in created_at then id order, take at most."""
|
||||
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
self.find_many = AsyncMock(side_effect=self._find_many)
|
||||
self.find_first = AsyncMock(return_value=None)
|
||||
self.update_many = AsyncMock(return_value=0)
|
||||
self.update = AsyncMock()
|
||||
|
||||
@staticmethod
|
||||
def _selected(row, where: dict) -> bool:
|
||||
window: Final = where["created_at"]
|
||||
cursor: Final = where.get("OR")
|
||||
return (
|
||||
row.created_at <= window["lte"]
|
||||
and ("gt" not in window or row.created_at > window["gt"])
|
||||
and (
|
||||
cursor is None
|
||||
or row.created_at > cursor[0]["created_at"]["gt"]
|
||||
or (row.created_at == cursor[1]["created_at"] and row.id > cursor[1]["id"]["gt"])
|
||||
)
|
||||
)
|
||||
|
||||
async def _find_many(self, *, where: dict, take: int, order: object) -> tuple:
|
||||
selected: Final = (row for row in self.rows if self._selected(row, where))
|
||||
return tuple(sorted(selected, key=lambda row: (row.created_at, row.id))[:take])
|
||||
|
||||
|
||||
class _FakeManagedObjectRow:
|
||||
"""One managed batch row the provider has finished but nothing has costed yet."""
|
||||
|
|
@ -2940,9 +3088,12 @@ class _FakeManagedObjectRow:
|
|||
class _FakeManagedObjectTable:
|
||||
"""A LiteLLM_ManagedObjectTable double backed by one real, mutable row.
|
||||
|
||||
It honours the batch_processed and status filters, so the poller's compare-and-swap
|
||||
and the managed-files deletion guard both read the same state a shared Postgres row
|
||||
would give them. Staleness sweeps (the only queries scoped by created_at) never match.
|
||||
It honours the batch_processed and status filters and the deletion guard's created_at cutoff, so the
|
||||
poller's compare-and-swap and the managed-files deletion guard both read the state a shared Postgres row
|
||||
would give them. The poll lanes' created_at windows are ignored, so every lane sees the row, and each
|
||||
read hands back a snapshot, so a write that lands mid-cycle is invisible to the row the poller holds.
|
||||
Writes journal "claim" or "release" when they flip batch_processed, "expire" when they mark the row
|
||||
stale_expired, and "finalize" otherwise.
|
||||
"""
|
||||
|
||||
def __init__(self, row: _FakeManagedObjectRow, journal: list):
|
||||
|
|
@ -2956,7 +3107,7 @@ class _FakeManagedObjectTable:
|
|||
def _matches(self, where: dict) -> bool:
|
||||
for key, value in where.items():
|
||||
if key == "created_at":
|
||||
if "lt" in value and not self.row.created_at < value["lt"]:
|
||||
if "gte" in value and not self.row.created_at >= value["gte"]:
|
||||
return False
|
||||
continue
|
||||
if key == "status":
|
||||
|
|
@ -2971,10 +3122,12 @@ class _FakeManagedObjectTable:
|
|||
async def _update_many(self, *, where: dict, data: dict) -> int:
|
||||
if not self._matches(where):
|
||||
return 0
|
||||
if "updated_at" in where:
|
||||
if "batch_processed" in data:
|
||||
self.journal.append("claim" if data["batch_processed"] else "release")
|
||||
elif data.get("status") == "stale_expired":
|
||||
self.journal.append("expire")
|
||||
else:
|
||||
self.journal.append("finalize")
|
||||
elif "batch_processed" in where:
|
||||
self.journal.append("claim" if data.get("batch_processed") else "release")
|
||||
for key, value in data.items():
|
||||
setattr(self.row, key, value)
|
||||
return 1
|
||||
|
|
@ -2985,7 +3138,7 @@ class _FakeManagedObjectTable:
|
|||
setattr(self.row, key, value)
|
||||
|
||||
async def _find_many(self, *, where: dict, take=None, order=None) -> list:
|
||||
return [self.row] if self._matches(where) else []
|
||||
return [copy(self.row)] if self._matches(where) else []
|
||||
|
||||
|
||||
class TestMultiPodBatchCostClaim:
|
||||
|
|
@ -3683,12 +3836,15 @@ class TestBatchLineItemsFromThePoller:
|
|||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def _provider(cls, journal: list):
|
||||
"""The provider's batch files by id, and a spend-log writer that journals each write."""
|
||||
def _provider(cls, journal: list, missing_file_error: Exception | None = None):
|
||||
"""The provider's batch files by id, and a spend-log writer that journals each write. Reading any other
|
||||
file raises missing_file_error, or the NotFoundError OpenAI answers with."""
|
||||
files = {"file-input-1": cls._INPUT, _CLAIM_OUTPUT_FILE_ID: cls._OUTPUT, "file-err-1": cls._ERROR}
|
||||
|
||||
async def _afile_content(**kwargs):
|
||||
journal.append(f"read {kwargs['file_id']}")
|
||||
if kwargs["file_id"] not in files and missing_file_error is not None:
|
||||
raise missing_file_error
|
||||
if kwargs["file_id"] not in files:
|
||||
from litellm.exceptions import NotFoundError
|
||||
|
||||
|
|
@ -3861,7 +4017,7 @@ class TestBatchLineItemsFromThePoller:
|
|||
await instance.check_batch_cost()
|
||||
await self._stored(instance)
|
||||
|
||||
assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
|
||||
assert journal == ["claim", "read file-err-1", "read file-input-1", "lines"]
|
||||
(payload,) = logger.payloads
|
||||
assert [(line["custom_id"], line["status"], line["response_cost"]) for line in payload["lines"]] == [
|
||||
("ok-1", "failure", 0.0)
|
||||
|
|
@ -3892,7 +4048,7 @@ class TestBatchLineItemsFromThePoller:
|
|||
await instance.check_batch_cost()
|
||||
await self._stored(instance)
|
||||
|
||||
assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
|
||||
assert journal == ["claim", "read file-err-1", "read file-input-1", "lines"]
|
||||
((line,),) = [payload["lines"] for payload in logger.payloads]
|
||||
assert line["request"] == json.loads(self._INPUT)
|
||||
|
||||
|
|
@ -3910,7 +4066,7 @@ class TestBatchLineItemsFromThePoller:
|
|||
await instance.check_batch_cost()
|
||||
await self._stored(instance)
|
||||
|
||||
assert journal == ["finalize", "read file-err-1", "read file-input-1"]
|
||||
assert journal == ["claim", "read file-err-1", "read file-input-1"]
|
||||
assert logger.payloads == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -3970,11 +4126,37 @@ class TestBatchLineItemsFromThePoller:
|
|||
await instance.check_batch_cost()
|
||||
await self._stored(instance)
|
||||
|
||||
assert journal == ["read file-out-deleted", "finalize", "read file-err-1", "read file-input-1", "lines"]
|
||||
assert journal == ["read file-out-deleted", "claim", "read file-err-1", "read file-input-1", "lines"]
|
||||
(payload,) = logger.payloads
|
||||
assert [(line["custom_id"], line["status"]) for line in payload["lines"]] == [("ok-1", "failure")]
|
||||
assert row.batch_processed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "missing_object", "retired"),
|
||||
(
|
||||
(404, "bucket/job-1/predictions.jsonl", True),
|
||||
(404, "bucket/job-2/predictions.jsonl", False),
|
||||
(500, "bucket/job-1/predictions.jsonl", False),
|
||||
),
|
||||
)
|
||||
async def test_a_storage_404_for_the_output_object_retires_the_batch_unbilled(
|
||||
self, status_code, missing_object, retired
|
||||
):
|
||||
"""Vertex AI reports a deleted gs:// output as a VertexAIError 404 that names the object without its
|
||||
scheme, and no later poll can read that object either."""
|
||||
from litellm.llms.vertex_ai.common_utils import VertexAIError
|
||||
|
||||
journal: list = []
|
||||
row, instance = self._setup(journal, output_file_id="gs://bucket/job-1/predictions.jsonl")
|
||||
missing: Final = VertexAIError(status_code=status_code, message=f"No such object: {missing_object}")
|
||||
|
||||
with self._provider(journal, missing_file_error=missing):
|
||||
await instance.check_batch_cost()
|
||||
|
||||
assert row.batch_processed is retired
|
||||
assert "bill" not in journal
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_terminal_batch_with_no_lines_sends_nothing(self, monkeypatch):
|
||||
import litellm
|
||||
|
|
@ -3988,7 +4170,7 @@ class TestBatchLineItemsFromThePoller:
|
|||
await instance.check_batch_cost()
|
||||
await self._stored(instance)
|
||||
|
||||
assert journal == ["finalize"]
|
||||
assert journal == ["claim"]
|
||||
assert logger.payloads == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -4088,16 +4270,19 @@ class TestBatchLineItemsFromThePoller:
|
|||
_, instance = self._setup(journal)
|
||||
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))
|
||||
|
||||
async def _an_earlier_batch_waits_to_store_its_lines():
|
||||
find_jobs: Final = instance._find_jobs
|
||||
|
||||
async def _an_earlier_batch_waits_to_store_its_lines(where):
|
||||
instance._store_batch_line_items_in_background(
|
||||
MagicMock(), MagicMock(), "model-123", "batch-earlier", billed=True
|
||||
)
|
||||
await asyncio.sleep(0.01)
|
||||
return await find_jobs(where)
|
||||
|
||||
async def _store(job, response, model_id, batch_id, *, billed):
|
||||
journal.append(f"store {batch_id}")
|
||||
|
||||
monkeypatch.setattr(instance, "_cleanup_stale_managed_objects", _an_earlier_batch_waits_to_store_its_lines)
|
||||
monkeypatch.setattr(instance, "_find_jobs", _an_earlier_batch_waits_to_store_its_lines)
|
||||
monkeypatch.setattr(instance, "_store_batch_line_items", _store)
|
||||
with self._provider(journal):
|
||||
await instance.check_batch_cost()
|
||||
|
|
@ -4345,23 +4530,317 @@ class TestCreatorTurnedOffMessageLogging:
|
|||
instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called()
|
||||
|
||||
|
||||
_PAST_THE_STALENESS_CUTOFF: Final = timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, hours=1)
|
||||
_INSIDE_THE_STALENESS_CUTOFF: Final = timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - timedelta(hours=1)
|
||||
|
||||
|
||||
def _provider_batch(status: str) -> LiteLLMBatch:
|
||||
return LiteLLMBatch(
|
||||
id="batch-456",
|
||||
status=status,
|
||||
output_file_id=None,
|
||||
error_file_id=None,
|
||||
created_at=1,
|
||||
completion_window="24h",
|
||||
endpoint="/v1/chat/completions",
|
||||
input_file_id="file-input-1",
|
||||
object="batch",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_completed_batch_remains_available_for_costing() -> None:
|
||||
async def test_an_unbilled_batch_past_the_staleness_cutoff_no_longer_blocks_deleting_its_input_file() -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.status = "completed"
|
||||
row.created_at = datetime(2000, 1, 1, tzinfo=timezone.utc)
|
||||
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
|
||||
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
|
||||
|
||||
await poller._cleanup_stale_managed_objects()
|
||||
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, "file-input-1")
|
||||
|
||||
assert row.batch_processed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("age", "file_field", "file_id"),
|
||||
(
|
||||
(_INSIDE_THE_STALENESS_CUTOFF, "input_file_id", "file-input-1"),
|
||||
(_PAST_THE_STALENESS_CUTOFF, "output_file_id", _CLAIM_OUTPUT_FILE_ID),
|
||||
(_PAST_THE_STALENESS_CUTOFF, "error_file_id", "file-error-1"),
|
||||
),
|
||||
ids=("input_inside_the_cutoff", "output_past_the_cutoff", "error_past_the_cutoff"),
|
||||
)
|
||||
async def test_an_unbilled_batch_still_blocks_deleting_a_file_its_cost_may_need(
|
||||
age: timedelta, file_field: str, file_id: str
|
||||
) -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.status = "completed"
|
||||
row.created_at = datetime.now(timezone.utc) - age
|
||||
row.file_object = json.dumps({"id": "batch-456", "status": "completed", file_field: file_id})
|
||||
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
|
||||
|
||||
with pytest.raises(HTTPException) as refused:
|
||||
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, file_id)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stale_expired_batch_no_longer_blocks_deleting_its_output_file() -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.status = "stale_expired"
|
||||
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
|
||||
|
||||
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
|
||||
|
||||
assert row.status == "stale_expired"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_still_billed() -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
|
||||
journal: Final = []
|
||||
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, journal)
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
|
||||
|
||||
with TestMultiPodBatchCostClaim._billing_patches(journal):
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert journal == ["fetch", "claim", "bill"]
|
||||
assert row.batch_processed is True
|
||||
|
||||
|
||||
_A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", llm_provider="openai", model="gpt-4")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("age", "db_status", "unified_object_id", "retrieve", "expected_status"),
|
||||
(
|
||||
(
|
||||
_PAST_THE_STALENESS_CUTOFF,
|
||||
"in_progress",
|
||||
_CLAIM_UNIFIED_BATCH_ID,
|
||||
[_provider_batch("in_progress")] * UNSETTLED_POLLS_BEFORE_EXPIRY,
|
||||
"in_progress",
|
||||
),
|
||||
(_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "in_progress"),
|
||||
(
|
||||
_PAST_THE_STALENESS_CUTOFF,
|
||||
"in_progress",
|
||||
_CLAIM_UNIFIED_BATCH_ID,
|
||||
litellm.InternalServerError(message="upstream error", llm_provider="openai", model="gpt-4"),
|
||||
"in_progress",
|
||||
),
|
||||
(
|
||||
_PAST_THE_STALENESS_CUTOFF,
|
||||
"in_progress",
|
||||
_CLAIM_UNIFIED_BATCH_ID,
|
||||
litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4"),
|
||||
"in_progress",
|
||||
),
|
||||
(_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "stale_expired"),
|
||||
(
|
||||
_PAST_THE_STALENESS_CUTOFF,
|
||||
"in_progress",
|
||||
"batch-raw-1",
|
||||
[_provider_batch("in_progress")] * UNSETTLED_POLLS_BEFORE_EXPIRY,
|
||||
"stale_expired",
|
||||
),
|
||||
(_INSIDE_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "in_progress"),
|
||||
(_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "completed"),
|
||||
(_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "completed"),
|
||||
),
|
||||
ids=(
|
||||
"still_running",
|
||||
"retrieve_timed_out",
|
||||
"provider_5xx",
|
||||
"provider_429",
|
||||
"key_rejected",
|
||||
"unroutable",
|
||||
"inside_the_cutoff",
|
||||
"finished_and_retrieve_timed_out",
|
||||
"finished_and_key_rejected",
|
||||
),
|
||||
)
|
||||
async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_staleness_cutoff(
|
||||
age: timedelta, db_status: str, unified_object_id: str, retrieve: object, expected_status: str
|
||||
) -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.created_at = datetime.now(timezone.utc) - age
|
||||
row.status = db_status
|
||||
row.unified_object_id = unified_object_id
|
||||
router: Final = TestMultiPodBatchCostClaim._router()
|
||||
router.aretrieve_batch.side_effect = retrieve
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router)
|
||||
|
||||
with TestMultiPodBatchCostClaim._billing_patches([]):
|
||||
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert row.status == expected_status
|
||||
assert row.batch_processed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unfinished_batch_past_the_staleness_cutoff_expires_only_after_consecutive_refused_retrieves() -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
|
||||
router: Final = TestMultiPodBatchCostClaim._router()
|
||||
router.aretrieve_batch.side_effect = [
|
||||
*([_A_REJECTED_KEY] * (UNSETTLED_POLLS_BEFORE_EXPIRY - 1)),
|
||||
_provider_batch("in_progress"),
|
||||
*([_A_REJECTED_KEY] * UNSETTLED_POLLS_BEFORE_EXPIRY),
|
||||
]
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router)
|
||||
statuses: Final = []
|
||||
|
||||
with TestMultiPodBatchCostClaim._billing_patches([]):
|
||||
for _ in range(2 * UNSETTLED_POLLS_BEFORE_EXPIRY):
|
||||
await poller.check_batch_cost()
|
||||
statuses.append(row.status)
|
||||
|
||||
assert statuses == [*(["in_progress"] * (2 * UNSETTLED_POLLS_BEFORE_EXPIRY - 1)), "stale_expired"]
|
||||
|
||||
|
||||
async def _a_vertex_job_that_no_longer_exists(*, model: str, batch_id: str, litellm_metadata: object) -> LiteLLMBatch:
|
||||
from litellm.llms.vertex_ai.common_utils import VertexAIError
|
||||
|
||||
raise VertexAIError(
|
||||
status_code=404,
|
||||
message='Error: 404 {"error":{"code":404,"message":"The BatchPredictionJob does not exist.","status":"NOT_FOUND"}}',
|
||||
)
|
||||
|
||||
|
||||
async def _a_finished_batch_whose_output_never_appeared(
|
||||
*, model: str, batch_id: str, litellm_metadata: object
|
||||
) -> LiteLLMBatch:
|
||||
return _provider_batch("completed")
|
||||
|
||||
|
||||
async def _an_output_file_the_key_cannot_read() -> None:
|
||||
raise litellm.PermissionDeniedError(
|
||||
message="forbidden",
|
||||
llm_provider="openai",
|
||||
model="gpt-4",
|
||||
response=httpx.Response(403, request=httpx.Request("GET", "https://api.openai.com/v1/files")),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("age", "retired"), ((_PAST_THE_STALENESS_CUTOFF, True), (_INSIDE_THE_STALENESS_CUTOFF, False)))
|
||||
async def test_a_finished_batch_whose_output_never_appeared_frees_its_files_once_past_the_staleness_cutoff(
|
||||
age: timedelta, retired: bool
|
||||
) -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.created_at = datetime.now(timezone.utc) - age
|
||||
row.status = "completed"
|
||||
router: Final = TestMultiPodBatchCostClaim._router()
|
||||
router.aretrieve_batch.side_effect = _a_finished_batch_whose_output_never_appeared
|
||||
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router)
|
||||
|
||||
with TestMultiPodBatchCostClaim._billing_patches([]):
|
||||
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert (row.status, row.batch_processed) == ("completed", retired)
|
||||
if retired:
|
||||
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
|
||||
else:
|
||||
with pytest.raises(HTTPException):
|
||||
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("db_status", ("in_progress", "completed"))
|
||||
async def test_a_batch_past_the_staleness_cutoff_whose_output_id_lags_is_billed_once_it_appears(db_status: str) -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
|
||||
row.status = db_status
|
||||
journal: Final = []
|
||||
router: Final = TestMultiPodBatchCostClaim._router()
|
||||
router.aretrieve_batch.side_effect = [
|
||||
*([_provider_batch("completed")] * (UNSETTLED_POLLS_BEFORE_EXPIRY - 1)),
|
||||
router.aretrieve_batch.return_value,
|
||||
]
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, journal), router)
|
||||
|
||||
with TestMultiPodBatchCostClaim._billing_patches(journal):
|
||||
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert journal == ["fetch", "claim", "bill"]
|
||||
assert row.batch_processed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("retrieve", "during_fetch"),
|
||||
(
|
||||
(_A_REJECTED_KEY, None),
|
||||
(_a_vertex_job_that_no_longer_exists, None),
|
||||
(None, _an_output_file_the_key_cannot_read),
|
||||
),
|
||||
ids=("key_rejected", "batch_404_without_its_id", "output_unreadable"),
|
||||
)
|
||||
async def test_a_finished_batch_the_provider_refuses_past_the_staleness_cutoff_bills_once_the_refusal_is_fixed(
|
||||
retrieve: object, during_fetch: object
|
||||
) -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
|
||||
row.status = "completed"
|
||||
router: Final = TestMultiPodBatchCostClaim._router()
|
||||
router.aretrieve_batch.side_effect = retrieve
|
||||
journal: Final = []
|
||||
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, journal)
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router)
|
||||
|
||||
with TestMultiPodBatchCostClaim._billing_patches(journal, during_fetch=during_fetch):
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert (row.status, row.batch_processed) == ("completed", False)
|
||||
with pytest.raises(HTTPException):
|
||||
await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
|
||||
|
||||
router.aretrieve_batch.side_effect = None
|
||||
with TestMultiPodBatchCostClaim._billing_patches(journal):
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert journal[-1] == "bill"
|
||||
assert row.batch_processed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_status_a_client_wrote_meanwhile() -> None:
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF
|
||||
router: Final = TestMultiPodBatchCostClaim._router()
|
||||
|
||||
async def _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_is_refused(
|
||||
*, model: str, batch_id: str, litellm_metadata: object
|
||||
) -> LiteLLMBatch:
|
||||
if router.aretrieve_batch.call_count == UNSETTLED_POLLS_BEFORE_EXPIRY:
|
||||
row.status = "completed"
|
||||
raise _A_REJECTED_KEY
|
||||
|
||||
router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_is_refused
|
||||
poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router)
|
||||
|
||||
with TestMultiPodBatchCostClaim._billing_patches([]):
|
||||
for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY):
|
||||
await poller.check_batch_cost()
|
||||
|
||||
assert row.status == "completed"
|
||||
assert row.batch_processed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
|
||||
async def test_stale_retirement_preserves_concurrent_progress(progress: str) -> None:
|
||||
async def test_stale_retirement_lands_unless_another_pod_claimed_the_row(progress: str) -> None:
|
||||
"""A client GET that rewrote the row while the poller held it moves updated_at but leaves the batch unbilled,
|
||||
so the retirement still lands."""
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
selected: Final = copy(row)
|
||||
if progress == "claimed":
|
||||
|
|
@ -4372,13 +4851,16 @@ async def test_stale_retirement_preserves_concurrent_progress(progress: str) ->
|
|||
|
||||
await poller._retire_job(selected, "provider batch disappeared")
|
||||
|
||||
assert row.batch_processed is (progress == "claimed")
|
||||
assert row.updated_at > selected.updated_at
|
||||
assert row.batch_processed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
|
||||
async def test_stale_terminal_response_preserves_concurrent_progress(progress: str) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("progress", "finalized", "expected_status"), [("claimed", False, "in_progress"), ("refreshed", True, "failed")]
|
||||
)
|
||||
async def test_stale_terminal_response_lands_unless_another_pod_claimed_the_row(
|
||||
progress: str, finalized: bool, expected_status: str
|
||||
) -> None:
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
row: Final = _FakeManagedObjectRow()
|
||||
|
|
@ -4399,11 +4881,10 @@ async def test_stale_terminal_response_preserves_concurrent_progress(progress: s
|
|||
status="failed",
|
||||
)
|
||||
|
||||
assert await poller._finalize_unbilled_terminal_job(selected, response) is False
|
||||
assert await poller._finalize_unbilled_terminal_job(selected, response) is finalized
|
||||
|
||||
assert row.file_object == '{"revision":"fresh"}'
|
||||
assert row.status == "in_progress"
|
||||
assert row.batch_processed is (progress == "claimed")
|
||||
assert row.status == expected_status
|
||||
assert row.batch_processed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -4457,6 +4938,23 @@ def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, mes
|
|||
assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("missing_key", "gone"), (("out/job-123/input.jsonl.out", True), ("out/job-123/other.jsonl.out", False))
|
||||
)
|
||||
def test_an_s3_no_such_key_for_the_output_object_is_permanent_loss(missing_key: str, gone: bool) -> None:
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
|
||||
|
||||
# S3 GetObject 404 body shape, checked live against a missing key on 2026-10-20
|
||||
error: Final = BedrockError(
|
||||
status_code=404,
|
||||
message='<?xml version="1.0" encoding="UTF-8"?>\n<Error><Code>NoSuchKey</Code>'
|
||||
f"<Message>The specified key does not exist.</Message><Key>{missing_key}</Key>"
|
||||
"<RequestId>R</RequestId><HostId>H</HostId></Error>",
|
||||
)
|
||||
assert CheckBatchCost._is_output_file_gone_at_provider(error, "s3://qa-bucket/out/job-123/input.jsonl.out") is gone
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_file_id", (None, ""))
|
||||
def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None:
|
||||
from litellm.exceptions import NotFoundError
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue