From f223197010efdfcb867c1dc5785b13dcc3612202 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:55:49 -0700 Subject: [PATCH 1/8] test(batches): cover the batch cost poller gaps left after #47190 --- .../common_utils/test_check_batch_cost.py | 320 ++++++++++++++---- 1 file changed, 258 insertions(+), 62 deletions(-) diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 33b8085e082..862399f66ab 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -12,7 +12,7 @@ import logging from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from copy import copy -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from hashlib import sha256 from queue import SimpleQueue from types import SimpleNamespace @@ -23,6 +23,7 @@ import pytest import respx from fastapi import HTTPException +from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS from litellm.types.router import LiteLLM_Params from litellm.types.utils import LiteLLMBatch @@ -154,23 +155,6 @@ class TestCheckBatchCost: llm_router=mock_llm_router, ) - @pytest.mark.asyncio - async def test_cleanup_scoped_to_batch_file_purpose(self, check_batch_cost_instance, mock_prisma_client): - """_cleanup_stale_managed_objects scopes its update to file_purpose='batch' only.""" - mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1) - # Return empty so the main poll loop exits immediately - mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[]) - - await check_batch_cost_instance.check_batch_cost() - - calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - stale_call = calls[0] - assert stale_call[1]["data"] == {"status": "stale_expired"} - where = stale_call[1]["where"] - assert where["file_purpose"] == "batch" - assert "stale_expired" in where["status"]["not_in"] - assert "created_at" in where - @pytest.mark.asyncio async def test_startup_probe_confirms_batch_processed_support(self, check_batch_cost_instance, mock_prisma_client): mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) @@ -744,7 +728,7 @@ class TestCheckBatchCost: await check_batch_cost_instance.check_batch_cost() - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, ( "Expected update() to be called exactly once for the completed job" ) update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] @@ -877,7 +861,7 @@ class TestCheckBatchCost: mock_update_database.assert_awaited_once() assert mock_update_database.call_args.kwargs["response_cost"] == 0.01 - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, ( "the job must still be marked processed once cost tracking succeeds" ) @@ -979,7 +963,7 @@ class TestCheckBatchCost: await check_batch_cost_instance.check_batch_cost() - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, ( f"Expected update() to be called exactly once for a {terminal_status} job" ) update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] @@ -1085,13 +1069,9 @@ class TestCheckBatchCost: assert owner.user_id == "user-1" assert owner.team_id == "team-1" - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2 + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1 update_call = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args - assert update_call.kwargs["where"] == { - "id": "job-terminal-mint-1", - "batch_processed": False, - "updated_at": mock_job.updated_at, - } + assert update_call.kwargs["where"] == {"id": "job-terminal-mint-1", "batch_processed": False} update_data = update_call.kwargs["data"] assert update_data["status"] == terminal_status assert update_data["batch_processed"] is True @@ -1156,7 +1136,7 @@ class TestCheckBatchCost: ) as mock_afile_content: await check_batch_cost_instance.check_batch_cost() - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, ( "a completed batch with no output file must be marked processed exactly once" ) update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] @@ -1245,6 +1225,7 @@ class TestCheckBatchCost: mock_job.id = "job-in-progress-1" mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA==" mock_job.created_by = "user-1" + mock_job.created_at = datetime.now(timezone.utc) mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job]) @@ -1375,7 +1356,7 @@ class TestCheckBatchCost: f"{terminal_status} batch with an output file must fetch results and be billed" ) mock_logging_obj.async_success_handler.assert_awaited_once() - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2 + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1 update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["batch_processed"] is True assert update_data["status"] == terminal_status, ( @@ -1548,7 +1529,7 @@ class TestCheckBatchCost: assert mock_afile_content.await_count == 1 mock_calculate.assert_not_awaited() - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 1, ( "a terminal batch with a 404ing output file must be retired, not retried forever" ) update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] @@ -1946,7 +1927,7 @@ class TestUnmanagedVertexRouting: mock_logging_obj.async_success_handler.assert_awaited_once() assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01 - assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2 + assert prisma.db.litellm_managedobjecttable.update_many.call_count == 1 update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["batch_processed"] is True assert update_data["status"] == "complete" @@ -2163,7 +2144,7 @@ class TestUnmanagedBedrockRouting: mock_logging_obj.async_success_handler.assert_awaited_once() assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02 - assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2 + assert prisma.db.litellm_managedobjecttable.update_many.call_count == 1 update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["batch_processed"] is True assert update_data["status"] == "complete" @@ -2725,6 +2706,7 @@ class TestPollPageStarvation: job.id = job_id job.unified_object_id = unified_object_id job.created_by = "user-1" + job.created_at = datetime.now(timezone.utc) return job @staticmethod @@ -2910,6 +2892,65 @@ class TestPollPageStarvation: prisma.db.litellm_managedobjecttable.update.assert_not_awaited() + @pytest.mark.asyncio + async def test_a_new_batch_is_polled_every_cycle_while_an_older_batch_stays_unbilled(self): + """An unbilled batch older than the arrival window used to take every other cycle away from the arrival + lane, so a new batch waited two poll intervals to be billed.""" + now: Final = datetime.now(timezone.utc) + rows: Final = tuple( + SimpleNamespace( + id=f"job-{name}", + unified_object_id=self._encode(f"litellm_proxy;model_id:model-123;llm_batch_id:batch_{name}"), + created_by="user-1", + created_at=now - age, + updated_at=now, + status="in_progress", + ) + for name, age in (("old", timedelta(days=3)), ("new", timedelta(hours=1))) + ) + prisma: Final = self._prisma([]) + prisma.db.litellm_managedobjecttable = _FakeManagedObjectPages(rows) + llm_router: Final = MagicMock() + llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress")) + poller: Final = self._instance(prisma, llm_router) + + async def _batches_polled_in_one_cycle() -> tuple[str, ...]: + llm_router.aretrieve_batch.reset_mock() + await poller.check_batch_cost() + return tuple(sorted(call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list)) + + assert [await _batches_polled_in_one_cycle() for _ in range(4)] == [("batch_new", "batch_old")] * 4 + + +class _FakeManagedObjectPages: + """A LiteLLM_ManagedObjectTable double over several unbilled rows that pages them the way Postgres would: + inside the lane's created_at window, after the keyset cursor, in created_at then id order, take at most.""" + + def __init__(self, rows): + self.rows = rows + self.find_many = AsyncMock(side_effect=self._find_many) + self.find_first = AsyncMock(return_value=None) + self.update_many = AsyncMock(return_value=0) + self.update = AsyncMock() + + @staticmethod + def _selected(row, where: dict) -> bool: + window: Final = where["created_at"] + cursor: Final = where.get("OR") + return ( + row.created_at <= window["lte"] + and ("gt" not in window or row.created_at > window["gt"]) + and ( + cursor is None + or row.created_at > cursor[0]["created_at"]["gt"] + or (row.created_at == cursor[1]["created_at"] and row.id > cursor[1]["id"]["gt"]) + ) + ) + + async def _find_many(self, *, where: dict, take: int, order: object) -> tuple: + selected: Final = (row for row in self.rows if self._selected(row, where)) + return tuple(sorted(selected, key=lambda row: (row.created_at, row.id))[:take]) + class _FakeManagedObjectRow: """One managed batch row the provider has finished but nothing has costed yet.""" @@ -2940,9 +2981,12 @@ class _FakeManagedObjectRow: class _FakeManagedObjectTable: """A LiteLLM_ManagedObjectTable double backed by one real, mutable row. - It honours the batch_processed and status filters, so the poller's compare-and-swap - and the managed-files deletion guard both read the same state a shared Postgres row - would give them. Staleness sweeps (the only queries scoped by created_at) never match. + It honours the batch_processed and status filters and the deletion guard's created_at cutoff, so the + poller's compare-and-swap and the managed-files deletion guard both read the state a shared Postgres row + would give them. The poll lanes' created_at windows are ignored, so every lane sees the row, and each + read hands back a snapshot, so a write that lands mid-cycle is invisible to the row the poller holds. + Writes journal "claim" or "release" when they flip batch_processed, "expire" when they mark the row + stale_expired, and "finalize" otherwise. """ def __init__(self, row: _FakeManagedObjectRow, journal: list): @@ -2956,7 +3000,7 @@ class _FakeManagedObjectTable: def _matches(self, where: dict) -> bool: for key, value in where.items(): if key == "created_at": - if "lt" in value and not self.row.created_at < value["lt"]: + if "gte" in value and not self.row.created_at >= value["gte"]: return False continue if key == "status": @@ -2971,10 +3015,12 @@ class _FakeManagedObjectTable: async def _update_many(self, *, where: dict, data: dict) -> int: if not self._matches(where): return 0 - if "updated_at" in where: + if "batch_processed" in data: + self.journal.append("claim" if data["batch_processed"] else "release") + elif data.get("status") == "stale_expired": + self.journal.append("expire") + else: self.journal.append("finalize") - elif "batch_processed" in where: - self.journal.append("claim" if data.get("batch_processed") else "release") for key, value in data.items(): setattr(self.row, key, value) return 1 @@ -2985,7 +3031,7 @@ class _FakeManagedObjectTable: setattr(self.row, key, value) async def _find_many(self, *, where: dict, take=None, order=None) -> list: - return [self.row] if self._matches(where) else [] + return [copy(self.row)] if self._matches(where) else [] class TestMultiPodBatchCostClaim: @@ -3683,12 +3729,15 @@ class TestBatchLineItemsFromThePoller: @classmethod @contextmanager - def _provider(cls, journal: list): - """The provider's batch files by id, and a spend-log writer that journals each write.""" + def _provider(cls, journal: list, missing_file_error: Exception | None = None): + """The provider's batch files by id, and a spend-log writer that journals each write. Reading any other + file raises missing_file_error, or the NotFoundError OpenAI answers with.""" files = {"file-input-1": cls._INPUT, _CLAIM_OUTPUT_FILE_ID: cls._OUTPUT, "file-err-1": cls._ERROR} async def _afile_content(**kwargs): journal.append(f"read {kwargs['file_id']}") + if kwargs["file_id"] not in files and missing_file_error is not None: + raise missing_file_error if kwargs["file_id"] not in files: from litellm.exceptions import NotFoundError @@ -3861,7 +3910,7 @@ class TestBatchLineItemsFromThePoller: await instance.check_batch_cost() await self._stored(instance) - assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"] + assert journal == ["claim", "read file-err-1", "read file-input-1", "lines"] (payload,) = logger.payloads assert [(line["custom_id"], line["status"], line["response_cost"]) for line in payload["lines"]] == [ ("ok-1", "failure", 0.0) @@ -3892,7 +3941,7 @@ class TestBatchLineItemsFromThePoller: await instance.check_batch_cost() await self._stored(instance) - assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"] + assert journal == ["claim", "read file-err-1", "read file-input-1", "lines"] ((line,),) = [payload["lines"] for payload in logger.payloads] assert line["request"] == json.loads(self._INPUT) @@ -3910,7 +3959,7 @@ class TestBatchLineItemsFromThePoller: await instance.check_batch_cost() await self._stored(instance) - assert journal == ["finalize", "read file-err-1", "read file-input-1"] + assert journal == ["claim", "read file-err-1", "read file-input-1"] assert logger.payloads == [] @pytest.mark.asyncio @@ -3970,11 +4019,37 @@ class TestBatchLineItemsFromThePoller: await instance.check_batch_cost() await self._stored(instance) - assert journal == ["read file-out-deleted", "finalize", "read file-err-1", "read file-input-1", "lines"] + assert journal == ["read file-out-deleted", "claim", "read file-err-1", "read file-input-1", "lines"] (payload,) = logger.payloads assert [(line["custom_id"], line["status"]) for line in payload["lines"]] == [("ok-1", "failure")] assert row.batch_processed is True + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("status_code", "missing_object", "retired"), + ( + (404, "bucket/job-1/predictions.jsonl", True), + (404, "bucket/job-2/predictions.jsonl", False), + (500, "bucket/job-1/predictions.jsonl", False), + ), + ) + async def test_a_storage_404_for_the_output_object_retires_the_batch_unbilled( + self, status_code, missing_object, retired + ): + """Vertex AI reports a deleted gs:// output as a VertexAIError 404 that names the object without its + scheme, and no later poll can read that object either.""" + from litellm.llms.vertex_ai.common_utils import VertexAIError + + journal: list = [] + row, instance = self._setup(journal, output_file_id="gs://bucket/job-1/predictions.jsonl") + missing: Final = VertexAIError(status_code=status_code, message=f"No such object: {missing_object}") + + with self._provider(journal, missing_file_error=missing): + await instance.check_batch_cost() + + assert row.batch_processed is retired + assert "bill" not in journal + @pytest.mark.asyncio async def test_a_terminal_batch_with_no_lines_sends_nothing(self, monkeypatch): import litellm @@ -3988,7 +4063,7 @@ class TestBatchLineItemsFromThePoller: await instance.check_batch_cost() await self._stored(instance) - assert journal == ["finalize"] + assert journal == ["claim"] assert logger.payloads == [] @pytest.mark.asyncio @@ -4088,16 +4163,19 @@ class TestBatchLineItemsFromThePoller: _, instance = self._setup(journal) monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),)) - async def _an_earlier_batch_waits_to_store_its_lines(): + find_jobs: Final = instance._find_jobs + + async def _an_earlier_batch_waits_to_store_its_lines(where): instance._store_batch_line_items_in_background( MagicMock(), MagicMock(), "model-123", "batch-earlier", billed=True ) await asyncio.sleep(0.01) + return await find_jobs(where) async def _store(job, response, model_id, batch_id, *, billed): journal.append(f"store {batch_id}") - monkeypatch.setattr(instance, "_cleanup_stale_managed_objects", _an_earlier_batch_waits_to_store_its_lines) + monkeypatch.setattr(instance, "_find_jobs", _an_earlier_batch_waits_to_store_its_lines) monkeypatch.setattr(instance, "_store_batch_line_items", _store) with self._provider(journal): await instance.check_batch_cost() @@ -4334,23 +4412,139 @@ class TestCreatorTurnedOffMessageLogging: instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called() +_PAST_THE_STALENESS_CUTOFF: Final = timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, hours=1) +_INSIDE_THE_STALENESS_CUTOFF: Final = timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - timedelta(hours=1) + + +def _provider_batch(status: str) -> LiteLLMBatch: + return LiteLLMBatch( + id="batch-456", + status=status, + output_file_id=None, + error_file_id=None, + created_at=1, + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id="file-input-1", + object="batch", + ) + + @pytest.mark.asyncio -async def test_old_completed_batch_remains_available_for_costing() -> None: +async def test_an_unbilled_batch_past_the_staleness_cutoff_no_longer_blocks_deleting_its_files() -> None: row: Final = _FakeManagedObjectRow() row.status = "completed" - row.created_at = datetime(2000, 1, 1, tzinfo=timezone.utc) + row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) - poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router()) - await poller._cleanup_stale_managed_objects() + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) assert row.batch_processed is False + + +@pytest.mark.asyncio +async def test_an_unbilled_batch_inside_the_staleness_cutoff_still_blocks_deleting_its_files() -> None: + row: Final = _FakeManagedObjectRow() + row.status = "completed" + row.created_at = datetime.now(timezone.utc) - _INSIDE_THE_STALENESS_CUTOFF + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + + with pytest.raises(HTTPException) as refused: + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) + + assert refused.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_still_billed() -> None: + row: Final = _FakeManagedObjectRow() + row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF + journal: Final = [] + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, journal) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router()) + + with TestMultiPodBatchCostClaim._billing_patches(journal): + await poller.check_batch_cost() + + assert journal == ["fetch", "claim", "bill"] + assert row.batch_processed is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("age", "db_status", "unified_object_id", "retrieve", "expected_status"), + ( + ( + _PAST_THE_STALENESS_CUTOFF, + "in_progress", + _CLAIM_UNIFIED_BATCH_ID, + [_provider_batch("in_progress")], + "stale_expired", + ), + ( + _PAST_THE_STALENESS_CUTOFF, + "in_progress", + _CLAIM_UNIFIED_BATCH_ID, + RuntimeError("timed out"), + "stale_expired", + ), + (_PAST_THE_STALENESS_CUTOFF, "in_progress", "batch-raw-1", [_provider_batch("in_progress")], "stale_expired"), + ( + _INSIDE_THE_STALENESS_CUTOFF, + "in_progress", + _CLAIM_UNIFIED_BATCH_ID, + [_provider_batch("in_progress")], + "in_progress", + ), + (_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "completed"), + ), + ids=("still_running", "retrieve_failed", "unroutable", "inside_the_cutoff", "finished_but_unbilled"), +) +async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_staleness_cutoff( + age: timedelta, db_status: str, unified_object_id: str, retrieve: object, expected_status: str +) -> None: + row: Final = _FakeManagedObjectRow() + row.created_at = datetime.now(timezone.utc) - age + row.status = db_status + row.unified_object_id = unified_object_id + router: Final = TestMultiPodBatchCostClaim._router() + router.aretrieve_batch.side_effect = retrieve + poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) + + with TestMultiPodBatchCostClaim._billing_patches([]): + await poller.check_batch_cost() + + assert row.status == expected_status + assert row.batch_processed is False + + +@pytest.mark.asyncio +async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_status_a_client_wrote_meanwhile() -> None: + row: Final = _FakeManagedObjectRow() + row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF + router: Final = TestMultiPodBatchCostClaim._router() + + async def _a_client_reads_the_finished_batch_after_the_poller_saw_it_running( + *, model: str, batch_id: str, litellm_metadata: object + ) -> LiteLLMBatch: + row.status = "completed" + return _provider_batch("in_progress") + + router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_after_the_poller_saw_it_running + poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) + + with TestMultiPodBatchCostClaim._billing_patches([]): + await poller.check_batch_cost() + assert row.status == "completed" + assert row.batch_processed is False @pytest.mark.asyncio @pytest.mark.parametrize("progress", ["claimed", "refreshed"]) -async def test_stale_retirement_preserves_concurrent_progress(progress: str) -> None: +async def test_stale_retirement_lands_unless_another_pod_claimed_the_row(progress: str) -> None: + """A client GET that rewrote the row while the poller held it moves updated_at but leaves the batch unbilled, + so the retirement still lands.""" row: Final = _FakeManagedObjectRow() selected: Final = copy(row) if progress == "claimed": @@ -4361,13 +4555,16 @@ async def test_stale_retirement_preserves_concurrent_progress(progress: str) -> await poller._retire_job(selected, "provider batch disappeared") - assert row.batch_processed is (progress == "claimed") - assert row.updated_at > selected.updated_at + assert row.batch_processed is True @pytest.mark.asyncio -@pytest.mark.parametrize("progress", ["claimed", "refreshed"]) -async def test_stale_terminal_response_preserves_concurrent_progress(progress: str) -> None: +@pytest.mark.parametrize( + ("progress", "finalized", "expected_status"), [("claimed", False, "in_progress"), ("refreshed", True, "failed")] +) +async def test_stale_terminal_response_lands_unless_another_pod_claimed_the_row( + progress: str, finalized: bool, expected_status: str +) -> None: from litellm.types.utils import LiteLLMBatch row: Final = _FakeManagedObjectRow() @@ -4388,11 +4585,10 @@ async def test_stale_terminal_response_preserves_concurrent_progress(progress: s status="failed", ) - assert await poller._finalize_unbilled_terminal_job(selected, response) is False + assert await poller._finalize_unbilled_terminal_job(selected, response) is finalized - assert row.file_object == '{"revision":"fresh"}' - assert row.status == "in_progress" - assert row.batch_processed is (progress == "claimed") + assert row.status == expected_status + assert row.batch_processed is True @pytest.mark.asyncio From 7bfac6e4894338e699972c9c44a490b495913cf4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:04:39 -0700 Subject: [PATCH 2/8] fix(batches): keep unbillable batches from blocking file deletes and new-batch polling The file delete check now ignores batches past MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, a provider 404 for the output object retires the batch, each poll page takes the arrival lane first, a batch expires only after the poller visits it past the cutoff, and the compare-and-swap matches batch_processed instead of updated_at --- .../proxy/common_utils/check_batch_cost.py | 100 ++++++++++-------- .../proxy/hooks/managed_files.py | 5 + 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index bd9440d4754..35d90b9ab11 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -151,27 +151,6 @@ class CheckBatchCost: return self.batch_processed_support_confirmed = True - async def _cleanup_stale_managed_objects(self) -> None: - """ - Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days - in non-terminal states as 'stale_expired'. These will never complete and - should not be polled. - """ - cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - result: Final = await _managed_object_table(self.prisma_client).update_many( - where={ - "file_purpose": "batch", - "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)}, - "created_at": {"lt": cutoff}, - }, - data={"status": "stale_expired"}, - ) - if result > 0: - verbose_proxy_logger.warning( - f"CheckBatchCost: marked {result} stale managed objects " - f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired" - ) - async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]": """Query batch jobs without the batch_processed filter (for older schemas).""" return await self._find_jobs( @@ -191,14 +170,20 @@ class CheckBatchCost: ) async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]: - arrivals: Final = self._poll_arrivals - page: Final = await self._find_job_page(where, arrivals=arrivals) - self._poll_arrivals = not arrivals if page else arrivals - if page: - return page - return await self._find_job_page(where, arrivals=not arrivals) + arrivals_first: Final = self._poll_arrivals + first: Final = await self._find_job_page(where, arrivals=arrivals_first, take=MAX_OBJECTS_PER_POLL_CYCLE) + rest: Final = await self._find_job_page( + where, arrivals=not arrivals_first, take=MAX_OBJECTS_PER_POLL_CYCLE - len(first) + ) + self._poll_arrivals = not (arrivals_first and len(first) == MAX_OBJECTS_PER_POLL_CYCLE) + first_ids: Final = frozenset(job.id for job in first) + return (*first, *(job for job in rest if job.id not in first_ids)) - async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]: + async def _find_job_page( + self, where: Mapping[str, object], *, arrivals: bool, take: int + ) -> Sequence[ManagedObjectRow]: + if take == 0: + return () cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor if cursor is None: if arrivals: @@ -224,10 +209,10 @@ class CheckBatchCost: else {} ), }, - take=MAX_OBJECTS_PER_POLL_CYCLE, + take=take, order=[{"created_at": "asc"}, {"id": "asc"}], ) - next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None + next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == take else None if arrivals: self._arrival_cursor = next_cursor else: @@ -235,11 +220,30 @@ class CheckBatchCost: return page def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]: - return { - "id": job.id, - "updated_at": job.updated_at, - **({"batch_processed": False} if self._has_batch_processed_column else {}), - } + if self._has_batch_processed_column: + return {"id": job.id, "batch_processed": False} + return {"id": job.id, "updated_at": job.updated_at} + + async def _expire_if_stale(self, job: ManagedObjectRow) -> None: + cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) + if job.status in TERMINAL_MANAGED_OBJECT_STATUSES or job.created_at >= cutoff: + return + try: + expired: Final = await _managed_object_table(self.prisma_client).update_many( + where={ + **self._unprocessed_snapshot(job), + "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)}, + }, + data={"status": "stale_expired"}, + ) + except Exception as db_err: + verbose_proxy_logger.error(f"CheckBatchCost: failed to mark stale job {job.id} as stale_expired: {db_err}") + return + if expired > 0: + verbose_proxy_logger.warning( + f"CheckBatchCost: job {job.id} is older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days " + "and still not finished, so it is marked stale_expired and no longer polled" + ) async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None: """ @@ -360,15 +364,22 @@ class CheckBatchCost: def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool: """A 404 naming the output file means there is nothing to fetch on this or any later poll: providers like Vertex AI advertise an output path for every batch, - including terminal ones that never wrote it. Any other failure may be - transient, so it keeps retrying until the staleness sweep bounds it.""" + including terminal ones that never wrote it. Storage backends name a gs:// or s3:// + object without its scheme. Any other failure may be transient, so it keeps retrying.""" import openai from litellm.exceptions import NotFoundError + from litellm.llms.base_llm.chat.transformation import BaseLLMException if not output_file_id: return False - return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id) + is_not_found: Final = isinstance(error, (NotFoundError, openai.NotFoundError)) or ( + isinstance(error, BaseLLMException) and error.status_code == 404 + ) + return is_not_found and ( + _error_names_resource(error, output_file_id) + or _error_names_resource(error, output_file_id.split("://", 1)[-1]) + ) async def _finalize_unbilled_terminal_job( self, job: "ManagedObjectRow", response: "LiteLLMBatch" @@ -1096,13 +1107,6 @@ class CheckBatchCost: processed_models: List[Tuple[Optional[str], Optional[str]]] = [] - try: - await self._cleanup_stale_managed_objects() - except Exception as cleanup_err: - verbose_proxy_logger.warning( - f"CheckBatchCost: stale cleanup failed (poll will continue): {cleanup_err}" - ) - # Look for all batches that have not yet been processed by CheckBatchCost. # self._has_batch_processed_column is cached after the first probe so that # older schemas don't pay a guaranteed-failing primary query + warning on @@ -1144,6 +1148,8 @@ class CheckBatchCost: if routing is None: if self._has_unified_id_without_model(job): await self._retire_job(job, "unified object id has no model id") + else: + await self._expire_if_stale(job) continue model_id, batch_id = routing @@ -1168,6 +1174,8 @@ class CheckBatchCost: prom_logger.record_check_batch_cost_error("provider_retrieval_error") if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id): await self._retire_job(job, f"batch {batch_id} no longer exists at the provider") + else: + await self._expire_if_stale(job) continue ## RETRIEVE THE BATCH JOB OUTPUT FILE @@ -1230,6 +1238,8 @@ class CheckBatchCost: ) continue await self._retire_unbilled_terminal_job(job, response, model_id, batch_id) + else: + await self._expire_if_stale(job) # Record polling run metrics (always, even if nothing was processed) if prom_logger: diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index bc35c4a0aba..0d446e33c7f 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -6,6 +6,7 @@ import base64 import json import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from datetime import datetime, timedelta, timezone from itertools import chain from types import MappingProxyType from typing import ( @@ -39,6 +40,7 @@ from litellm.constants import ( BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS, BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, + MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, MAX_FILE_LIST_LIMIT, PENDING_RESPONSE_STATUSES, ) @@ -2379,6 +2381,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "file_purpose": "batch", "batch_processed": False, "status": {"not_in": ["failed", "expired", "cancelled"]}, + "created_at": { + "gte": datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) + }, }, take=MAX_MATCHES_TO_RETURN, order={"created_at": "desc"}, From 3029e70f40562f8032da6268750170c1d3dd5150 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:48:54 -0700 Subject: [PATCH 3/8] test(batches): cover the full arrival page and a failed expiry write in the batch cost poller Old batches still get polled every other cycle while new batches fill the page, old batches that fill a page never push a new batch out after boot, and a failed stale_expired write no longer stops the rest of the page from being polled --- .../common_utils/test_check_batch_cost.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 862399f66ab..db39f241483 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -2921,6 +2921,79 @@ class TestPollPageStarvation: assert [await _batches_polled_in_one_cycle() for _ in range(4)] == [("batch_new", "batch_old")] * 4 + def _unbilled_rows(self, ages: dict[str, timedelta]) -> tuple[SimpleNamespace, ...]: + now: Final = datetime.now(timezone.utc) + return tuple( + SimpleNamespace( + id=f"job-{name}", + unified_object_id=self._encode(f"litellm_proxy;model_id:model-123;llm_batch_id:batch_{name}"), + created_by="user-1", + created_at=now - age, + updated_at=now, + status="in_progress", + ) + for name, age in ages.items() + ) + + async def _cycles_on_a_two_row_page(self, monkeypatch, ages: dict[str, timedelta], cycles: int): + from litellm_enterprise.proxy.common_utils import check_batch_cost + + monkeypatch.setattr(check_batch_cost, "MAX_OBJECTS_PER_POLL_CYCLE", 2) + prisma: Final = self._prisma([]) + prisma.db.litellm_managedobjecttable = _FakeManagedObjectPages(self._unbilled_rows(ages)) + llm_router: Final = MagicMock() + llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress")) + poller: Final = self._instance(prisma, llm_router) + + async def _batches_polled_in_one_cycle() -> frozenset[str]: + llm_router.aretrieve_batch.reset_mock() + await poller.check_batch_cost() + return frozenset(call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list) + + return [await _batches_polled_in_one_cycle() for _ in range(cycles)] + + @pytest.mark.asyncio + async def test_an_old_batch_is_polled_every_other_cycle_while_new_batches_fill_the_page(self, monkeypatch): + new_batches: Final = {f"new{n}": timedelta(hours=5 - n) for n in range(1, 5)} + polled: Final = await self._cycles_on_a_two_row_page( + monkeypatch, {"old": timedelta(days=3), **new_batches}, cycles=8 + ) + + assert all("batch_old" in pair[0] | pair[1] for pair in zip(polled, polled[1:])) + assert all(f"batch_{name}" in frozenset().union(*polled[:4]) for name in new_batches) + + @pytest.mark.asyncio + async def test_old_batches_that_fill_the_page_never_push_a_new_batch_out_after_boot(self, monkeypatch): + polled: Final = await self._cycles_on_a_two_row_page( + monkeypatch, + {"old1": timedelta(days=3, hours=1), "old2": timedelta(days=3), "new": timedelta(hours=1)}, + cycles=6, + ) + + assert all("batch_new" in cycle for cycle in polled[1:]) + assert frozenset({"batch_old1", "batch_old2"}) <= frozenset().union(*polled[1:]) + + @pytest.mark.asyncio + async def test_a_failed_expiry_write_leaves_the_rest_of_the_page_polled(self): + rows: Final = self._unbilled_rows( + {"old": timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, hours=1), "new": timedelta(hours=1)} + ) + table: Final = _FakeManagedObjectPages(rows) + table.update_many = AsyncMock(side_effect=RuntimeError("database unavailable")) + prisma: Final = self._prisma([]) + prisma.db.litellm_managedobjecttable = table + llm_router: Final = MagicMock() + llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress")) + poller: Final = self._instance(prisma, llm_router) + + await poller.check_batch_cost() + + assert [call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list] == [ + "batch_old", + "batch_new", + ] + assert table.update_many.await_count == 1 + class _FakeManagedObjectPages: """A LiteLLM_ManagedObjectTable double over several unbilled rows that pages them the way Postgres would: From 74d2c12dc006eae1959ec56c25a90f63288c1482 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:21:39 -0700 Subject: [PATCH 4/8] fix(batches): share the poll page between old and new batches and hold output files until billed Each lane now gets at least half of the poll page, a lane with nothing left gives its half to the other, and a lane read that resumed mid-lane wraps to the start, so neither old nor new batches can starve the other. The delete guard applies the 7 day cutoff to input files only. Output and error files stay locked until the batch is billed or retired, and stale_expired rows no longer lock anything. Only a lasting provider failure expires a batch past the cutoff, so a timeout or a 5xx no longer gives up on a batch that may still bill. Past the cutoff a lasting failure, a lasting cost-tracking failure, or an output file id that never appeared retires a finished row instead of polling it forever. --- .../proxy/common_utils/check_batch_cost.py | 55 +++++- .../proxy/hooks/managed_files.py | 8 +- .../proxy/test_file_deletion_blocking.py | 3 +- .../common_utils/test_check_batch_cost.py | 165 ++++++++++++++++-- 4 files changed, 200 insertions(+), 31 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 35d90b9ab11..5fe38725780 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -90,6 +90,10 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li return TeamRepository(prisma_client).table +def _unique_jobs(jobs: Sequence[ManagedObjectRow]) -> tuple[ManagedObjectRow, ...]: + return tuple({job.id: job for job in jobs}.values()) + + def _row_metadata(row: object) -> dict[str, object]: metadata: Final = getattr(row, "metadata", None) return cast(dict[str, object], metadata) if isinstance(metadata, dict) else {} # cast-ok: prisma Json object column @@ -170,20 +174,37 @@ class CheckBatchCost: ) async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]: - arrivals_first: Final = self._poll_arrivals - first: Final = await self._find_job_page(where, arrivals=arrivals_first, take=MAX_OBJECTS_PER_POLL_CYCLE) - rest: Final = await self._find_job_page( - where, arrivals=not arrivals_first, take=MAX_OBJECTS_PER_POLL_CYCLE - len(first) + lead_arrivals: Final = self._poll_arrivals + self._poll_arrivals = not lead_arrivals + lead_share: Final = MAX_OBJECTS_PER_POLL_CYCLE - MAX_OBJECTS_PER_POLL_CYCLE // 2 + lead: Final = await self._find_job_page(where, arrivals=lead_arrivals, take=lead_share) + trail: Final = await self._find_job_page( + where, arrivals=not lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead) ) - self._poll_arrivals = not (arrivals_first and len(first) == MAX_OBJECTS_PER_POLL_CYCLE) - first_ids: Final = frozenset(job.id for job in first) - return (*first, *(job for job in rest if job.id not in first_ids)) + spill: Final = ( + await self._find_job_page( + where, arrivals=lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead) - len(trail) + ) + if len(lead) == lead_share + else () + ) + return _unique_jobs((*lead, *trail, *spill)) async def _find_job_page( self, where: Mapping[str, object], *, arrivals: bool, take: int ) -> Sequence[ManagedObjectRow]: if take == 0: return () + resumed_mid_lane: Final = (self._arrival_cursor if arrivals else self._poll_cursor) is not None + page: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take) + if len(page) == take or not resumed_mid_lane: + return page + wrapped: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take - len(page)) + return _unique_jobs((*page, *wrapped)) + + async def _find_job_page_from_cursor( + self, where: Mapping[str, object], *, arrivals: bool, take: int + ) -> Sequence[ManagedObjectRow]: cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor if cursor is None: if arrivals: @@ -226,7 +247,10 @@ class CheckBatchCost: async def _expire_if_stale(self, job: ManagedObjectRow) -> None: cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - if job.status in TERMINAL_MANAGED_OBJECT_STATUSES or job.created_at >= cutoff: + if job.created_at >= cutoff: + return + if job.status in TERMINAL_MANAGED_OBJECT_STATUSES: + await self._retire_job(job, f"unbilled {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days after it was created") return try: expired: Final = await _managed_object_table(self.prisma_client).update_many( @@ -353,6 +377,16 @@ class CheckBatchCost: return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id) + @staticmethod + def _is_lasting_provider_failure(error: Exception) -> bool: + import openai + + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + if not isinstance(error, (BaseLLMException, openai.APIStatusError)): + return False + return not litellm.should_retry(error.status_code) + def _batch_deployment_exists(self, model_id: str) -> bool: """A 404 only proves the batch is gone when it came from the batch's own deployment. Once that deployment leaves the router, default fallbacks can @@ -1174,7 +1208,7 @@ class CheckBatchCost: prom_logger.record_check_batch_cost_error("provider_retrieval_error") if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id): await self._retire_job(job, f"batch {batch_id} no longer exists at the provider") - else: + elif self._is_lasting_provider_failure(e): await self._expire_if_stale(job) continue @@ -1203,6 +1237,8 @@ class CheckBatchCost: f"(job {job.id}): {tracking_err}" ) self._record_error(prom_logger, "cost_tracking_error") + if self._is_lasting_provider_failure(tracking_err): + await self._expire_if_stale(job) continue if tracked is None: continue @@ -1236,6 +1272,7 @@ class CheckBatchCost: f"CheckBatchCost: batch {batch_id} is completed but its output file id " f"has not appeared yet; leaving job {job.id} for the next poll cycle" ) + await self._expire_if_stale(job) continue await self._retire_unbilled_terminal_job(job, response, model_id, batch_id) else: diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 0d446e33c7f..16bb493f78c 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -2380,15 +2380,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): where={ "file_purpose": "batch", "batch_processed": False, - "status": {"not_in": ["failed", "expired", "cancelled"]}, - "created_at": { - "gte": datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - }, + "status": {"not_in": ["failed", "expired", "cancelled", "stale_expired"]}, }, take=MAX_MATCHES_TO_RETURN, order={"created_at": "desc"}, ) + input_file_cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) referencing_batches: Final[list[dict[str, object]]] = [] for batch in batches: try: @@ -2401,7 +2399,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Extract file IDs from batch # Batches typically reference the unified file ID in input_file_id # Output and error files are generated by the provider - input_file_id = batch_data.get("input_file_id") + input_file_id = batch_data.get("input_file_id") if batch.created_at >= input_file_cutoff else None output_file_id = batch_data.get("output_file_id") error_file_id = batch_data.get("error_file_id") diff --git a/tests/unit/enterprise/proxy/test_file_deletion_blocking.py b/tests/unit/enterprise/proxy/test_file_deletion_blocking.py index b7acdad35a1..b02a74a9bfb 100644 --- a/tests/unit/enterprise/proxy/test_file_deletion_blocking.py +++ b/tests/unit/enterprise/proxy/test_file_deletion_blocking.py @@ -10,6 +10,7 @@ This ensures cost tracking is not disrupted by premature file deletion. import base64 import json +from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -50,7 +51,7 @@ def _make_batch_db_record( mock_batch.status = status mock_batch.file_object = json.dumps(file_object) mock_batch.created_by = created_by - mock_batch.created_at = 1700000000 + mock_batch.created_at = datetime.now(timezone.utc) return mock_batch diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index db39f241483..ca3da103767 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -19,10 +19,12 @@ from types import SimpleNamespace from typing import TYPE_CHECKING, Final, Literal from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx from fastapi import HTTPException +import litellm from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS from litellm.types.router import LiteLLM_Params from litellm.types.utils import LiteLLMBatch @@ -235,7 +237,7 @@ class TestCheckBatchCost: fallback_where = calls[1][1]["where"] assert "batch_processed" not in fallback_where assert "stale_expired" in fallback_where["status"]["not_in"] - assert calls[1][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE + assert calls[2][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE # Column absence is now cached — next call should go straight to fallback assert check_batch_cost_instance._has_batch_processed_column is False assert check_batch_cost_instance.batch_processed_support_confirmed is False @@ -1185,6 +1187,7 @@ class TestCheckBatchCost: b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456" ).decode() mock_job.created_by = "user-1" + mock_job.created_at = datetime.now(timezone.utc) assert check_batch_cost_instance._has_batch_processed_column is True mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job]) @@ -2935,10 +2938,10 @@ class TestPollPageStarvation: for name, age in ages.items() ) - async def _cycles_on_a_two_row_page(self, monkeypatch, ages: dict[str, timedelta], cycles: int): + async def _poll_cycles(self, monkeypatch, ages: dict[str, timedelta], cycles: int, page_size: int = 2): from litellm_enterprise.proxy.common_utils import check_batch_cost - monkeypatch.setattr(check_batch_cost, "MAX_OBJECTS_PER_POLL_CYCLE", 2) + monkeypatch.setattr(check_batch_cost, "MAX_OBJECTS_PER_POLL_CYCLE", page_size) prisma: Final = self._prisma([]) prisma.db.litellm_managedobjecttable = _FakeManagedObjectPages(self._unbilled_rows(ages)) llm_router: Final = MagicMock() @@ -2953,25 +2956,52 @@ class TestPollPageStarvation: return [await _batches_polled_in_one_cycle() for _ in range(cycles)] @pytest.mark.asyncio - async def test_an_old_batch_is_polled_every_other_cycle_while_new_batches_fill_the_page(self, monkeypatch): + async def test_an_old_batch_is_polled_every_cycle_while_new_batches_fill_the_page(self, monkeypatch): new_batches: Final = {f"new{n}": timedelta(hours=5 - n) for n in range(1, 5)} - polled: Final = await self._cycles_on_a_two_row_page( + polled: Final = await self._poll_cycles( monkeypatch, {"old": timedelta(days=3), **new_batches}, cycles=8 ) - assert all("batch_old" in pair[0] | pair[1] for pair in zip(polled, polled[1:])) + assert all("batch_old" in cycle for cycle in polled) assert all(f"batch_{name}" in frozenset().union(*polled[:4]) for name in new_batches) @pytest.mark.asyncio - async def test_old_batches_that_fill_the_page_never_push_a_new_batch_out_after_boot(self, monkeypatch): - polled: Final = await self._cycles_on_a_two_row_page( + async def test_old_batches_that_fill_the_page_never_push_a_new_batch_out(self, monkeypatch): + polled: Final = await self._poll_cycles( monkeypatch, {"old1": timedelta(days=3, hours=1), "old2": timedelta(days=3), "new": timedelta(hours=1)}, cycles=6, ) - assert all("batch_new" in cycle for cycle in polled[1:]) - assert frozenset({"batch_old1", "batch_old2"}) <= frozenset().union(*polled[1:]) + assert all("batch_new" in cycle for cycle in polled) + assert frozenset({"batch_old1", "batch_old2"}) <= frozenset().union(*polled[:2]) + + @pytest.mark.asyncio + async def test_old_and_new_batches_each_get_half_the_page_when_both_overflow_it(self, monkeypatch): + ages: Final = { + **{f"old{n}": timedelta(days=3, hours=n) for n in range(3)}, + **{f"new{n}": timedelta(hours=n + 1) for n in range(3)}, + } + polled: Final = await self._poll_cycles(monkeypatch, ages, cycles=6) + + assert [sorted(batch_id[len("batch_") : -1] for batch_id in cycle) for cycle in polled] == [["new", "old"]] * 6 + assert frozenset().union(*polled) == frozenset(f"batch_{name}" for name in ages) + + @pytest.mark.asyncio + async def test_old_and_new_batches_take_turns_on_a_one_row_page(self, monkeypatch): + polled: Final = await self._poll_cycles( + monkeypatch, {"old": timedelta(days=3), "new": timedelta(hours=1)}, cycles=4, page_size=1 + ) + + assert polled == [frozenset({"batch_old"}), frozenset({"batch_new"})] * 2 + + @pytest.mark.asyncio + async def test_new_batches_fill_the_page_when_no_old_batch_needs_polling(self, monkeypatch): + polled: Final = await self._poll_cycles( + monkeypatch, {f"new{n}": timedelta(hours=n + 1) for n in range(3)}, cycles=4 + ) + + assert [len(cycle) for cycle in polled] == [2] * 4 @pytest.mark.asyncio async def test_a_failed_expiry_write_leaves_the_rest_of_the_page_polled(self): @@ -4504,30 +4534,53 @@ def _provider_batch(status: str) -> LiteLLMBatch: @pytest.mark.asyncio -async def test_an_unbilled_batch_past_the_staleness_cutoff_no_longer_blocks_deleting_its_files() -> None: +async def test_an_unbilled_batch_past_the_staleness_cutoff_no_longer_blocks_deleting_its_input_file() -> None: row: Final = _FakeManagedObjectRow() row.status = "completed" row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) - await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, "file-input-1") assert row.batch_processed is False @pytest.mark.asyncio -async def test_an_unbilled_batch_inside_the_staleness_cutoff_still_blocks_deleting_its_files() -> None: +@pytest.mark.parametrize( + ("age", "file_field", "file_id"), + ( + (_INSIDE_THE_STALENESS_CUTOFF, "input_file_id", "file-input-1"), + (_PAST_THE_STALENESS_CUTOFF, "output_file_id", _CLAIM_OUTPUT_FILE_ID), + (_PAST_THE_STALENESS_CUTOFF, "error_file_id", "file-error-1"), + ), + ids=("input_inside_the_cutoff", "output_past_the_cutoff", "error_past_the_cutoff"), +) +async def test_an_unbilled_batch_still_blocks_deleting_a_file_its_cost_may_need( + age: timedelta, file_field: str, file_id: str +) -> None: row: Final = _FakeManagedObjectRow() row.status = "completed" - row.created_at = datetime.now(timezone.utc) - _INSIDE_THE_STALENESS_CUTOFF + row.created_at = datetime.now(timezone.utc) - age + row.file_object = json.dumps({"id": "batch-456", "status": "completed", file_field: file_id}) prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) with pytest.raises(HTTPException) as refused: - await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, file_id) assert refused.value.status_code == 400 +@pytest.mark.asyncio +async def test_a_stale_expired_batch_no_longer_blocks_deleting_its_output_file() -> None: + row: Final = _FakeManagedObjectRow() + row.status = "stale_expired" + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) + + assert row.status == "stale_expired" + + @pytest.mark.asyncio async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_still_billed() -> None: row: Final = _FakeManagedObjectRow() @@ -4559,6 +4612,20 @@ async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_s "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), + "in_progress", + ), + ( + _PAST_THE_STALENESS_CUTOFF, + "in_progress", + _CLAIM_UNIFIED_BATCH_ID, + litellm.InternalServerError(message="upstream error", llm_provider="openai", model="gpt-4"), + "in_progress", + ), + ( + _PAST_THE_STALENESS_CUTOFF, + "in_progress", + _CLAIM_UNIFIED_BATCH_ID, + litellm.AuthenticationError(message="invalid api key", llm_provider="openai", model="gpt-4"), "stale_expired", ), (_PAST_THE_STALENESS_CUTOFF, "in_progress", "batch-raw-1", [_provider_batch("in_progress")], "stale_expired"), @@ -4571,7 +4638,15 @@ async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_s ), (_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "completed"), ), - ids=("still_running", "retrieve_failed", "unroutable", "inside_the_cutoff", "finished_but_unbilled"), + ids=( + "still_running", + "retrieve_timed_out", + "provider_5xx", + "key_rejected", + "unroutable", + "inside_the_cutoff", + "finished_but_unbilled", + ), ) async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_staleness_cutoff( age: timedelta, db_status: str, unified_object_id: str, retrieve: object, expected_status: str @@ -4591,6 +4666,64 @@ async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_stale assert row.batch_processed is False +async def _a_vertex_job_that_no_longer_exists(*, model: str, batch_id: str, litellm_metadata: object) -> LiteLLMBatch: + from litellm.llms.vertex_ai.common_utils import VertexAIError + + raise VertexAIError( + status_code=404, + message='Error: 404 {"error":{"code":404,"message":"The BatchPredictionJob does not exist.","status":"NOT_FOUND"}}', + ) + + +async def _a_finished_batch_whose_output_never_appeared( + *, model: str, batch_id: str, litellm_metadata: object +) -> LiteLLMBatch: + return _provider_batch("completed") + + +async def _an_output_file_the_key_cannot_read() -> None: + raise litellm.PermissionDeniedError( + message="forbidden", + llm_provider="openai", + model="gpt-4", + response=httpx.Response(403, request=httpx.Request("GET", "https://api.openai.com/v1/files")), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("age", "retired"), ((_PAST_THE_STALENESS_CUTOFF, True), (_INSIDE_THE_STALENESS_CUTOFF, False))) +@pytest.mark.parametrize( + ("retrieve", "during_fetch"), + ( + (_a_vertex_job_that_no_longer_exists, None), + (_a_finished_batch_whose_output_never_appeared, None), + (None, _an_output_file_the_key_cannot_read), + ), + ids=("batch_gone_without_its_id", "output_never_appeared", "output_unreadable"), +) +async def test_a_finished_batch_the_poller_cannot_bill_frees_its_files_once_past_the_staleness_cutoff( + age: timedelta, retired: bool, retrieve: object, during_fetch: object +) -> None: + row: Final = _FakeManagedObjectRow() + row.created_at = datetime.now(timezone.utc) - age + row.status = "completed" + router: Final = TestMultiPodBatchCostClaim._router() + if retrieve is not None: + router.aretrieve_batch.side_effect = retrieve + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router) + + with TestMultiPodBatchCostClaim._billing_patches([], during_fetch=during_fetch): + await poller.check_batch_cost() + + assert (row.status, row.batch_processed) == ("completed", retired) + if retired: + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) + else: + with pytest.raises(HTTPException): + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) + + @pytest.mark.asyncio async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_status_a_client_wrote_meanwhile() -> None: row: Final = _FakeManagedObjectRow() From df1f5eda73318001f23a4770e04b104dff5faef6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:34:28 -0700 Subject: [PATCH 5/8] fix(batches): retry refused finished batches past 7 days, expire unfinished ones on a failed retrieve A finished batch the provider refuses after 7 days (rotated key, bucket 403, a 404 that does not name the batch) now keeps retrying so its spend is written once the config is fixed, instead of retiring unbilled. A batch the provider still reports running keeps polling. A batch that is not finished in the database becomes stale_expired once a retrieve fails after 7 days, so an unreachable provider is not polled forever. An S3 NoSuchKey for the output object now counts as gone. --- .../proxy/common_utils/check_batch_cost.py | 41 +++---- .../common_utils/test_check_batch_cost.py | 111 +++++++++++------- 2 files changed, 87 insertions(+), 65 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 2954e1b4ce4..c788df5d9eb 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: def _error_names_resource(error: Exception, resource_id: str) -> bool: - return resource_id in re.findall(r"""[^\s"'`]+""", str(error)) + return resource_id in re.findall(r"""[^\s"'`<>]+""", str(error)) CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost" @@ -241,13 +241,19 @@ class CheckBatchCost: return {"id": job.id, "batch_processed": False} return {"id": job.id, "updated_at": job.updated_at} + @staticmethod + def _is_past_staleness_cutoff(job: ManagedObjectRow) -> bool: + return job.created_at < datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) + async def _expire_if_stale(self, job: ManagedObjectRow) -> None: - cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - if job.created_at >= cutoff: + if not self._is_past_staleness_cutoff(job): return if job.status in TERMINAL_MANAGED_OBJECT_STATUSES: await self._retire_job(job, f"unbilled {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days after it was created") return + await self._expire_unfinished(job) + + async def _expire_unfinished(self, job: ManagedObjectRow) -> None: try: expired: Final = await _managed_object_table(self.prisma_client).update_many( where={ @@ -373,16 +379,6 @@ class CheckBatchCost: return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id) - @staticmethod - def _is_lasting_provider_failure(error: Exception) -> bool: - import openai - - from litellm.llms.base_llm.chat.transformation import BaseLLMException - - if not isinstance(error, (BaseLLMException, openai.APIStatusError)): - return False - return not litellm.should_retry(error.status_code) - def _batch_deployment_exists(self, model_id: str) -> bool: """A 404 only proves the batch is gone when it came from the batch's own deployment. Once that deployment leaves the router, default fallbacks can @@ -394,8 +390,8 @@ class CheckBatchCost: def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool: """A 404 naming the output file means there is nothing to fetch on this or any later poll: providers like Vertex AI advertise an output path for every batch, - including terminal ones that never wrote it. Storage backends name a gs:// or s3:// - object without its scheme. Any other failure may be transient, so it keeps retrying.""" + including terminal ones that never wrote it. GCS names a gs:// object without its scheme, + and S3 names an s3:// object by its key alone. Any other failure keeps retrying.""" import openai from litellm.exceptions import NotFoundError @@ -406,10 +402,9 @@ class CheckBatchCost: is_not_found: Final = isinstance(error, (NotFoundError, openai.NotFoundError)) or ( isinstance(error, BaseLLMException) and error.status_code == 404 ) - return is_not_found and ( - _error_names_resource(error, output_file_id) - or _error_names_resource(error, output_file_id.split("://", 1)[-1]) - ) + object_path: Final = output_file_id.split("://", 1)[-1] + names: Final = (output_file_id, object_path, object_path.split("/", 1)[-1]) + return is_not_found and any(_error_names_resource(error, name) for name in names) async def _finalize_unbilled_terminal_job( self, job: "ManagedObjectRow", response: "LiteLLMBatch" @@ -1206,8 +1201,8 @@ class CheckBatchCost: prom_logger.record_check_batch_cost_error("provider_retrieval_error") if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id): await self._retire_job(job, f"batch {batch_id} no longer exists at the provider") - elif self._is_lasting_provider_failure(e): - await self._expire_if_stale(job) + elif self._is_past_staleness_cutoff(job) and job.status not in TERMINAL_MANAGED_OBJECT_STATUSES: + await self._expire_unfinished(job) continue ## RETRIEVE THE BATCH JOB OUTPUT FILE @@ -1235,8 +1230,6 @@ class CheckBatchCost: f"(job {job.id}): {tracking_err}" ) self._record_error(prom_logger, "cost_tracking_error") - if self._is_lasting_provider_failure(tracking_err): - await self._expire_if_stale(job) continue if tracked is None: continue @@ -1273,8 +1266,6 @@ class CheckBatchCost: await self._expire_if_stale(job) continue await self._retire_unbilled_terminal_job(job, response, model_id, batch_id) - else: - await self._expire_if_stale(job) # Record polling run metrics (always, even if nothing was processed) if prom_logger: diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index ca3da103767..9f1eb406f06 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -3013,7 +3013,7 @@ class TestPollPageStarvation: prisma: Final = self._prisma([]) prisma.db.litellm_managedobjecttable = table llm_router: Final = MagicMock() - llm_router.aretrieve_batch = AsyncMock(return_value=MagicMock(status="in_progress")) + llm_router.aretrieve_batch = AsyncMock(side_effect=RuntimeError("timed out")) poller: Final = self._instance(prisma, llm_router) await poller.check_batch_cost() @@ -4596,6 +4596,9 @@ async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_s assert row.batch_processed is True +_A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", llm_provider="openai", model="gpt-4") + + @pytest.mark.asyncio @pytest.mark.parametrize( ("age", "db_status", "unified_object_id", "retrieve", "expected_status"), @@ -4605,38 +4608,21 @@ async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_s "in_progress", _CLAIM_UNIFIED_BATCH_ID, [_provider_batch("in_progress")], - "stale_expired", - ), - ( - _PAST_THE_STALENESS_CUTOFF, - "in_progress", - _CLAIM_UNIFIED_BATCH_ID, - RuntimeError("timed out"), "in_progress", ), + (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "stale_expired"), ( _PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, litellm.InternalServerError(message="upstream error", llm_provider="openai", model="gpt-4"), - "in_progress", - ), - ( - _PAST_THE_STALENESS_CUTOFF, - "in_progress", - _CLAIM_UNIFIED_BATCH_ID, - litellm.AuthenticationError(message="invalid api key", llm_provider="openai", model="gpt-4"), "stale_expired", ), + (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "stale_expired"), (_PAST_THE_STALENESS_CUTOFF, "in_progress", "batch-raw-1", [_provider_batch("in_progress")], "stale_expired"), - ( - _INSIDE_THE_STALENESS_CUTOFF, - "in_progress", - _CLAIM_UNIFIED_BATCH_ID, - [_provider_batch("in_progress")], - "in_progress", - ), + (_INSIDE_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "in_progress"), (_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "completed"), + (_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "completed"), ), ids=( "still_running", @@ -4645,7 +4631,8 @@ async def test_a_batch_past_the_staleness_cutoff_that_the_provider_finished_is_s "key_rejected", "unroutable", "inside_the_cutoff", - "finished_but_unbilled", + "finished_and_retrieve_timed_out", + "finished_and_key_rejected", ), ) async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_staleness_cutoff( @@ -4692,28 +4679,18 @@ async def _an_output_file_the_key_cannot_read() -> None: @pytest.mark.asyncio @pytest.mark.parametrize(("age", "retired"), ((_PAST_THE_STALENESS_CUTOFF, True), (_INSIDE_THE_STALENESS_CUTOFF, False))) -@pytest.mark.parametrize( - ("retrieve", "during_fetch"), - ( - (_a_vertex_job_that_no_longer_exists, None), - (_a_finished_batch_whose_output_never_appeared, None), - (None, _an_output_file_the_key_cannot_read), - ), - ids=("batch_gone_without_its_id", "output_never_appeared", "output_unreadable"), -) -async def test_a_finished_batch_the_poller_cannot_bill_frees_its_files_once_past_the_staleness_cutoff( - age: timedelta, retired: bool, retrieve: object, during_fetch: object +async def test_a_finished_batch_whose_output_never_appeared_frees_its_files_once_past_the_staleness_cutoff( + age: timedelta, retired: bool ) -> None: row: Final = _FakeManagedObjectRow() row.created_at = datetime.now(timezone.utc) - age row.status = "completed" router: Final = TestMultiPodBatchCostClaim._router() - if retrieve is not None: - router.aretrieve_batch.side_effect = retrieve + router.aretrieve_batch.side_effect = _a_finished_batch_whose_output_never_appeared prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router) - with TestMultiPodBatchCostClaim._billing_patches([], during_fetch=during_fetch): + with TestMultiPodBatchCostClaim._billing_patches([]): await poller.check_batch_cost() assert (row.status, row.batch_processed) == ("completed", retired) @@ -4724,19 +4701,56 @@ async def test_a_finished_batch_the_poller_cannot_bill_frees_its_files_once_past await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("retrieve", "during_fetch"), + ( + (_A_REJECTED_KEY, None), + (_a_vertex_job_that_no_longer_exists, None), + (None, _an_output_file_the_key_cannot_read), + ), + ids=("key_rejected", "batch_404_without_its_id", "output_unreadable"), +) +async def test_a_finished_batch_the_provider_refuses_past_the_staleness_cutoff_bills_once_the_refusal_is_fixed( + retrieve: object, during_fetch: object +) -> None: + row: Final = _FakeManagedObjectRow() + row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF + row.status = "completed" + router: Final = TestMultiPodBatchCostClaim._router() + router.aretrieve_batch.side_effect = retrieve + journal: Final = [] + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, journal) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router) + + with TestMultiPodBatchCostClaim._billing_patches(journal, during_fetch=during_fetch): + await poller.check_batch_cost() + + assert (row.status, row.batch_processed) == ("completed", False) + with pytest.raises(HTTPException): + await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) + + router.aretrieve_batch.side_effect = None + with TestMultiPodBatchCostClaim._billing_patches(journal): + await poller.check_batch_cost() + + assert journal[-1] == "bill" + assert row.batch_processed is True + + @pytest.mark.asyncio async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_status_a_client_wrote_meanwhile() -> None: row: Final = _FakeManagedObjectRow() row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF router: Final = TestMultiPodBatchCostClaim._router() - async def _a_client_reads_the_finished_batch_after_the_poller_saw_it_running( + async def _a_client_reads_the_finished_batch_while_the_poller_retrieve_times_out( *, model: str, batch_id: str, litellm_metadata: object ) -> LiteLLMBatch: row.status = "completed" - return _provider_batch("in_progress") + raise RuntimeError("timed out") - router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_after_the_poller_saw_it_running + router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_while_the_poller_retrieve_times_out poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) with TestMultiPodBatchCostClaim._billing_patches([]): @@ -4848,6 +4862,23 @@ def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, mes assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True +@pytest.mark.parametrize( + ("missing_key", "gone"), (("out/job-123/input.jsonl.out", True), ("out/job-123/other.jsonl.out", False)) +) +def test_an_s3_no_such_key_for_the_output_object_is_permanent_loss(missing_key: str, gone: bool) -> None: + from litellm.llms.bedrock.common_utils import BedrockError + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + + # S3 GetObject 404 body shape, checked live against a missing key on 2026-10-20 + error: Final = BedrockError( + status_code=404, + message='\nNoSuchKey' + f"The specified key does not exist.{missing_key}" + "RH", + ) + assert CheckBatchCost._is_output_file_gone_at_provider(error, "s3://qa-bucket/out/job-123/input.jsonl.out") is gone + + @pytest.mark.parametrize("output_file_id", (None, "")) def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None: from litellm.exceptions import NotFoundError From c7e0a4f5299a78562ae9cf9f84bdf0661aeac004 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:26:30 -0700 Subject: [PATCH 6/8] fix(batches): expire an unfinished old batch only after 3 failed retrieves in a row --- .../proxy/common_utils/check_batch_cost.py | 19 ++++++- .../common_utils/test_check_batch_cost.py | 55 +++++++++++++++---- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index c788df5d9eb..98cdc4f1ddd 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -47,6 +47,7 @@ def _error_names_resource(error: Exception, resource_id: str) -> bool: CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost" RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48) +FAILED_RETRIEVES_BEFORE_EXPIRY: Final = 3 PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = ( "completed", @@ -120,6 +121,7 @@ class CheckBatchCost: self._arrival_cursor: tuple[datetime, str] | None = None self._arrival_before: datetime = self._poll_before self._poll_arrivals: bool = False + self._failed_retrieves: Mapping[str, int] = MappingProxyType({}) self._line_item_stores: Final[set[asyncio.Task[None]]] = set() self._line_item_store_turn: Final = asyncio.Lock() self._poll_idle: Final = asyncio.Event() @@ -253,6 +255,20 @@ class CheckBatchCost: return await self._expire_unfinished(job) + async def _expire_unfinished_after_repeated_failures(self, job: ManagedObjectRow) -> None: + failures: Final = self._failed_retrieves.get(job.id, 0) + 1 + if failures < FAILED_RETRIEVES_BEFORE_EXPIRY: + self._failed_retrieves = MappingProxyType({**self._failed_retrieves, job.id: failures}) + return + self._forget_failed_retrieves(job) + await self._expire_unfinished(job) + + def _forget_failed_retrieves(self, job: ManagedObjectRow) -> None: + if job.id in self._failed_retrieves: + self._failed_retrieves = MappingProxyType( + {job_id: count for job_id, count in self._failed_retrieves.items() if job_id != job.id} + ) + async def _expire_unfinished(self, job: ManagedObjectRow) -> None: try: expired: Final = await _managed_object_table(self.prisma_client).update_many( @@ -1202,8 +1218,9 @@ class CheckBatchCost: if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id): await self._retire_job(job, f"batch {batch_id} no longer exists at the provider") elif self._is_past_staleness_cutoff(job) and job.status not in TERMINAL_MANAGED_OBJECT_STATUSES: - await self._expire_unfinished(job) + await self._expire_unfinished_after_repeated_failures(job) continue + self._forget_failed_retrieves(job) ## RETRIEVE THE BATCH JOB OUTPUT FILE if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None: diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 9f1eb406f06..67da8e9cfe4 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -28,6 +28,7 @@ import litellm from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS from litellm.types.router import LiteLLM_Params from litellm.types.utils import LiteLLMBatch +from litellm_enterprise.proxy.common_utils.check_batch_cost import FAILED_RETRIEVES_BEFORE_EXPIRY if TYPE_CHECKING: from litellm.batches.batch_utils import BatchCostUsageResult @@ -3016,13 +3017,16 @@ class TestPollPageStarvation: llm_router.aretrieve_batch = AsyncMock(side_effect=RuntimeError("timed out")) poller: Final = self._instance(prisma, llm_router) + for _ in range(FAILED_RETRIEVES_BEFORE_EXPIRY - 1): + await poller.check_batch_cost() + llm_router.aretrieve_batch.reset_mock() await poller.check_batch_cost() - assert [call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list] == [ + assert table.update_many.await_count == 1 + assert {call.kwargs["batch_id"] for call in llm_router.aretrieve_batch.await_args_list} == { "batch_old", "batch_new", - ] - assert table.update_many.await_count == 1 + } class _FakeManagedObjectPages: @@ -4607,7 +4611,7 @@ _A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", _PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, - [_provider_batch("in_progress")], + [_provider_batch("in_progress")] * FAILED_RETRIEVES_BEFORE_EXPIRY, "in_progress", ), (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "stale_expired"), @@ -4619,7 +4623,13 @@ _A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", "stale_expired", ), (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "stale_expired"), - (_PAST_THE_STALENESS_CUTOFF, "in_progress", "batch-raw-1", [_provider_batch("in_progress")], "stale_expired"), + ( + _PAST_THE_STALENESS_CUTOFF, + "in_progress", + "batch-raw-1", + [_provider_batch("in_progress")] * FAILED_RETRIEVES_BEFORE_EXPIRY, + "stale_expired", + ), (_INSIDE_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "in_progress"), (_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "completed"), (_PAST_THE_STALENESS_CUTOFF, "completed", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "completed"), @@ -4647,12 +4657,35 @@ async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_stale poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) with TestMultiPodBatchCostClaim._billing_patches([]): - await poller.check_batch_cost() + for _ in range(FAILED_RETRIEVES_BEFORE_EXPIRY): + await poller.check_batch_cost() assert row.status == expected_status assert row.batch_processed is False +@pytest.mark.asyncio +async def test_an_unfinished_batch_past_the_staleness_cutoff_expires_only_after_consecutive_failed_retrieves() -> None: + row: Final = _FakeManagedObjectRow() + row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF + timeout: Final = RuntimeError("timed out") + router: Final = TestMultiPodBatchCostClaim._router() + router.aretrieve_batch.side_effect = [ + *([timeout] * (FAILED_RETRIEVES_BEFORE_EXPIRY - 1)), + _provider_batch("in_progress"), + *([timeout] * FAILED_RETRIEVES_BEFORE_EXPIRY), + ] + poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) + statuses: Final = [] + + with TestMultiPodBatchCostClaim._billing_patches([]): + for _ in range(2 * FAILED_RETRIEVES_BEFORE_EXPIRY): + await poller.check_batch_cost() + statuses.append(row.status) + + assert statuses == [*(["in_progress"] * (2 * FAILED_RETRIEVES_BEFORE_EXPIRY - 1)), "stale_expired"] + + async def _a_vertex_job_that_no_longer_exists(*, model: str, batch_id: str, litellm_metadata: object) -> LiteLLMBatch: from litellm.llms.vertex_ai.common_utils import VertexAIError @@ -4744,17 +4777,19 @@ async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_statu row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF router: Final = TestMultiPodBatchCostClaim._router() - async def _a_client_reads_the_finished_batch_while_the_poller_retrieve_times_out( + async def _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_times_out( *, model: str, batch_id: str, litellm_metadata: object ) -> LiteLLMBatch: - row.status = "completed" + if router.aretrieve_batch.call_count == FAILED_RETRIEVES_BEFORE_EXPIRY: + row.status = "completed" raise RuntimeError("timed out") - router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_while_the_poller_retrieve_times_out + router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_times_out poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) with TestMultiPodBatchCostClaim._billing_patches([]): - await poller.check_batch_cost() + for _ in range(FAILED_RETRIEVES_BEFORE_EXPIRY): + await poller.check_batch_cost() assert row.status == "completed" assert row.batch_processed is False From 76af66dcaac1e794b80a2f6a467cd03297242483 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:43:49 -0700 Subject: [PATCH 7/8] fix(batches): keep retrying an unfinished old batch through timeouts, 429 and 5xx --- .../proxy/common_utils/check_batch_cost.py | 16 ++++++++++- .../common_utils/test_check_batch_cost.py | 27 ++++++++++++------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 98cdc4f1ddd..8aa5ba2f2fe 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -395,6 +395,16 @@ class CheckBatchCost: return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id) + @staticmethod + def _is_lasting_provider_failure(error: Exception) -> bool: + import openai + + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + return isinstance(error, (BaseLLMException, openai.APIStatusError)) and not litellm.should_retry( + error.status_code + ) + def _batch_deployment_exists(self, model_id: str) -> bool: """A 404 only proves the batch is gone when it came from the batch's own deployment. Once that deployment leaves the router, default fallbacks can @@ -1217,7 +1227,11 @@ class CheckBatchCost: prom_logger.record_check_batch_cost_error("provider_retrieval_error") if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id): await self._retire_job(job, f"batch {batch_id} no longer exists at the provider") - elif self._is_past_staleness_cutoff(job) and job.status not in TERMINAL_MANAGED_OBJECT_STATUSES: + elif ( + self._is_past_staleness_cutoff(job) + and job.status not in TERMINAL_MANAGED_OBJECT_STATUSES + and self._is_lasting_provider_failure(e) + ): await self._expire_unfinished_after_repeated_failures(job) continue self._forget_failed_retrieves(job) diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 67da8e9cfe4..69d4fd1a752 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -3014,7 +3014,7 @@ class TestPollPageStarvation: prisma: Final = self._prisma([]) prisma.db.litellm_managedobjecttable = table llm_router: Final = MagicMock() - llm_router.aretrieve_batch = AsyncMock(side_effect=RuntimeError("timed out")) + llm_router.aretrieve_batch = AsyncMock(side_effect=_A_REJECTED_KEY) poller: Final = self._instance(prisma, llm_router) for _ in range(FAILED_RETRIEVES_BEFORE_EXPIRY - 1): @@ -4614,13 +4614,20 @@ _A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", [_provider_batch("in_progress")] * FAILED_RETRIEVES_BEFORE_EXPIRY, "in_progress", ), - (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "stale_expired"), + (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "in_progress"), ( _PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, litellm.InternalServerError(message="upstream error", llm_provider="openai", model="gpt-4"), - "stale_expired", + "in_progress", + ), + ( + _PAST_THE_STALENESS_CUTOFF, + "in_progress", + _CLAIM_UNIFIED_BATCH_ID, + litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4"), + "in_progress", ), (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "stale_expired"), ( @@ -4638,6 +4645,7 @@ _A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", "still_running", "retrieve_timed_out", "provider_5xx", + "provider_429", "key_rejected", "unroutable", "inside_the_cutoff", @@ -4665,15 +4673,14 @@ async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_stale @pytest.mark.asyncio -async def test_an_unfinished_batch_past_the_staleness_cutoff_expires_only_after_consecutive_failed_retrieves() -> None: +async def test_an_unfinished_batch_past_the_staleness_cutoff_expires_only_after_consecutive_refused_retrieves() -> None: row: Final = _FakeManagedObjectRow() row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF - timeout: Final = RuntimeError("timed out") router: Final = TestMultiPodBatchCostClaim._router() router.aretrieve_batch.side_effect = [ - *([timeout] * (FAILED_RETRIEVES_BEFORE_EXPIRY - 1)), + *([_A_REJECTED_KEY] * (FAILED_RETRIEVES_BEFORE_EXPIRY - 1)), _provider_batch("in_progress"), - *([timeout] * FAILED_RETRIEVES_BEFORE_EXPIRY), + *([_A_REJECTED_KEY] * FAILED_RETRIEVES_BEFORE_EXPIRY), ] poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) statuses: Final = [] @@ -4777,14 +4784,14 @@ async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_statu row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF router: Final = TestMultiPodBatchCostClaim._router() - async def _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_times_out( + async def _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_is_refused( *, model: str, batch_id: str, litellm_metadata: object ) -> LiteLLMBatch: if router.aretrieve_batch.call_count == FAILED_RETRIEVES_BEFORE_EXPIRY: row.status = "completed" - raise RuntimeError("timed out") + raise _A_REJECTED_KEY - router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_times_out + router.aretrieve_batch.side_effect = _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_is_refused poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) with TestMultiPodBatchCostClaim._billing_patches([]): From 33e270063257c149db2341b208693d20731899b3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:25:37 -0700 Subject: [PATCH 8/8] fix(batches): wait 3 polls before expiring an old batch whose output id lags --- .../proxy/common_utils/check_batch_cost.py | 59 +++++++++++-------- .../common_utils/test_check_batch_cost.py | 47 +++++++++++---- 2 files changed, 69 insertions(+), 37 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 8aa5ba2f2fe..bc36f7f8cab 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -47,7 +47,7 @@ def _error_names_resource(error: Exception, resource_id: str) -> bool: CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost" RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48) -FAILED_RETRIEVES_BEFORE_EXPIRY: Final = 3 +UNSETTLED_POLLS_BEFORE_EXPIRY: Final = 3 PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = ( "completed", @@ -121,7 +121,7 @@ class CheckBatchCost: self._arrival_cursor: tuple[datetime, str] | None = None self._arrival_before: datetime = self._poll_before self._poll_arrivals: bool = False - self._failed_retrieves: Mapping[str, int] = MappingProxyType({}) + self._unsettled_polls: Mapping[str, int] = MappingProxyType({}) self._line_item_stores: Final[set[asyncio.Task[None]]] = set() self._line_item_store_turn: Final = asyncio.Lock() self._poll_idle: Final = asyncio.Event() @@ -255,20 +255,32 @@ class CheckBatchCost: return await self._expire_unfinished(job) - async def _expire_unfinished_after_repeated_failures(self, job: ManagedObjectRow) -> None: - failures: Final = self._failed_retrieves.get(job.id, 0) + 1 - if failures < FAILED_RETRIEVES_BEFORE_EXPIRY: - self._failed_retrieves = MappingProxyType({**self._failed_retrieves, job.id: failures}) + async def _expire_after_repeated_unsettled_polls(self, job: ManagedObjectRow) -> None: + unsettled: Final = self._unsettled_polls.get(job.id, 0) + 1 + if unsettled < UNSETTLED_POLLS_BEFORE_EXPIRY: + self._unsettled_polls = MappingProxyType({**self._unsettled_polls, job.id: unsettled}) return - self._forget_failed_retrieves(job) - await self._expire_unfinished(job) + self._forget_unsettled_polls(job) + await self._expire_if_stale(job) - def _forget_failed_retrieves(self, job: ManagedObjectRow) -> None: - if job.id in self._failed_retrieves: - self._failed_retrieves = MappingProxyType( - {job_id: count for job_id, count in self._failed_retrieves.items() if job_id != job.id} + def _forget_unsettled_polls(self, job: ManagedObjectRow) -> None: + if job.id in self._unsettled_polls: + self._unsettled_polls = MappingProxyType( + {job_id: count for job_id, count in self._unsettled_polls.items() if job_id != job.id} ) + @staticmethod + def _is_completed_with_output_id_pending(response: "LiteLLMBatch") -> bool: + from litellm.proxy.openai_files_endpoints.common_utils import ( + _completed_batch_safe_to_retire, + ) + + return ( + response.status in ("completed", "complete") + and response.output_file_id is None + and not _completed_batch_safe_to_retire(response) + ) + async def _expire_unfinished(self, job: ManagedObjectRow) -> None: try: expired: Final = await _managed_object_table(self.prisma_client).update_many( @@ -1232,9 +1244,17 @@ class CheckBatchCost: and job.status not in TERMINAL_MANAGED_OBJECT_STATUSES and self._is_lasting_provider_failure(e) ): - await self._expire_unfinished_after_repeated_failures(job) + await self._expire_after_repeated_unsettled_polls(job) continue - self._forget_failed_retrieves(job) + if self._is_completed_with_output_id_pending(response): + verbose_proxy_logger.info( + f"CheckBatchCost: batch {batch_id} is completed but its output file id " + f"has not appeared yet; leaving job {job.id} for the next poll cycle" + ) + if self._is_past_staleness_cutoff(job): + await self._expire_after_repeated_unsettled_polls(job) + continue + self._forget_unsettled_polls(job) ## RETRIEVE THE BATCH JOB OUTPUT FILE if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None: @@ -1285,17 +1305,6 @@ class CheckBatchCost: self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True) elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES: - from litellm.proxy.openai_files_endpoints.common_utils import ( - _completed_batch_safe_to_retire, - ) - - if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response): - verbose_proxy_logger.info( - f"CheckBatchCost: batch {batch_id} is completed but its output file id " - f"has not appeared yet; leaving job {job.id} for the next poll cycle" - ) - await self._expire_if_stale(job) - continue await self._retire_unbilled_terminal_job(job, response, model_id, batch_id) # Record polling run metrics (always, even if nothing was processed) diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 69d4fd1a752..89c81896cf6 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -28,7 +28,7 @@ import litellm from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS from litellm.types.router import LiteLLM_Params from litellm.types.utils import LiteLLMBatch -from litellm_enterprise.proxy.common_utils.check_batch_cost import FAILED_RETRIEVES_BEFORE_EXPIRY +from litellm_enterprise.proxy.common_utils.check_batch_cost import UNSETTLED_POLLS_BEFORE_EXPIRY if TYPE_CHECKING: from litellm.batches.batch_utils import BatchCostUsageResult @@ -3017,7 +3017,7 @@ class TestPollPageStarvation: llm_router.aretrieve_batch = AsyncMock(side_effect=_A_REJECTED_KEY) poller: Final = self._instance(prisma, llm_router) - for _ in range(FAILED_RETRIEVES_BEFORE_EXPIRY - 1): + for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY - 1): await poller.check_batch_cost() llm_router.aretrieve_batch.reset_mock() await poller.check_batch_cost() @@ -4611,7 +4611,7 @@ _A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", _PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, - [_provider_batch("in_progress")] * FAILED_RETRIEVES_BEFORE_EXPIRY, + [_provider_batch("in_progress")] * UNSETTLED_POLLS_BEFORE_EXPIRY, "in_progress", ), (_PAST_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, RuntimeError("timed out"), "in_progress"), @@ -4634,7 +4634,7 @@ _A_REJECTED_KEY: Final = litellm.AuthenticationError(message="invalid api key", _PAST_THE_STALENESS_CUTOFF, "in_progress", "batch-raw-1", - [_provider_batch("in_progress")] * FAILED_RETRIEVES_BEFORE_EXPIRY, + [_provider_batch("in_progress")] * UNSETTLED_POLLS_BEFORE_EXPIRY, "stale_expired", ), (_INSIDE_THE_STALENESS_CUTOFF, "in_progress", _CLAIM_UNIFIED_BATCH_ID, _A_REJECTED_KEY, "in_progress"), @@ -4665,7 +4665,7 @@ async def test_a_batch_the_poller_cannot_settle_expires_only_once_past_the_stale poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) with TestMultiPodBatchCostClaim._billing_patches([]): - for _ in range(FAILED_RETRIEVES_BEFORE_EXPIRY): + for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY): await poller.check_batch_cost() assert row.status == expected_status @@ -4678,19 +4678,19 @@ async def test_an_unfinished_batch_past_the_staleness_cutoff_expires_only_after_ row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF router: Final = TestMultiPodBatchCostClaim._router() router.aretrieve_batch.side_effect = [ - *([_A_REJECTED_KEY] * (FAILED_RETRIEVES_BEFORE_EXPIRY - 1)), + *([_A_REJECTED_KEY] * (UNSETTLED_POLLS_BEFORE_EXPIRY - 1)), _provider_batch("in_progress"), - *([_A_REJECTED_KEY] * FAILED_RETRIEVES_BEFORE_EXPIRY), + *([_A_REJECTED_KEY] * UNSETTLED_POLLS_BEFORE_EXPIRY), ] poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) statuses: Final = [] with TestMultiPodBatchCostClaim._billing_patches([]): - for _ in range(2 * FAILED_RETRIEVES_BEFORE_EXPIRY): + for _ in range(2 * UNSETTLED_POLLS_BEFORE_EXPIRY): await poller.check_batch_cost() statuses.append(row.status) - assert statuses == [*(["in_progress"] * (2 * FAILED_RETRIEVES_BEFORE_EXPIRY - 1)), "stale_expired"] + assert statuses == [*(["in_progress"] * (2 * UNSETTLED_POLLS_BEFORE_EXPIRY - 1)), "stale_expired"] async def _a_vertex_job_that_no_longer_exists(*, model: str, batch_id: str, litellm_metadata: object) -> LiteLLMBatch: @@ -4731,7 +4731,8 @@ async def test_a_finished_batch_whose_output_never_appeared_frees_its_files_once poller: Final = TestMultiPodBatchCostClaim._instance(prisma, router) with TestMultiPodBatchCostClaim._billing_patches([]): - await poller.check_batch_cost() + for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY): + await poller.check_batch_cost() assert (row.status, row.batch_processed) == ("completed", retired) if retired: @@ -4741,6 +4742,28 @@ async def test_a_finished_batch_whose_output_never_appeared_frees_its_files_once await TestMultiPodBatchCostClaim._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) +@pytest.mark.asyncio +@pytest.mark.parametrize("db_status", ("in_progress", "completed")) +async def test_a_batch_past_the_staleness_cutoff_whose_output_id_lags_is_billed_once_it_appears(db_status: str) -> None: + row: Final = _FakeManagedObjectRow() + row.created_at = datetime.now(timezone.utc) - _PAST_THE_STALENESS_CUTOFF + row.status = db_status + journal: Final = [] + router: Final = TestMultiPodBatchCostClaim._router() + router.aretrieve_batch.side_effect = [ + *([_provider_batch("completed")] * (UNSETTLED_POLLS_BEFORE_EXPIRY - 1)), + router.aretrieve_batch.return_value, + ] + poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, journal), router) + + with TestMultiPodBatchCostClaim._billing_patches(journal): + for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY): + await poller.check_batch_cost() + + assert journal == ["fetch", "claim", "bill"] + assert row.batch_processed is True + + @pytest.mark.asyncio @pytest.mark.parametrize( ("retrieve", "during_fetch"), @@ -4787,7 +4810,7 @@ async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_statu async def _a_client_reads_the_finished_batch_while_the_last_poller_retrieve_is_refused( *, model: str, batch_id: str, litellm_metadata: object ) -> LiteLLMBatch: - if router.aretrieve_batch.call_count == FAILED_RETRIEVES_BEFORE_EXPIRY: + if router.aretrieve_batch.call_count == UNSETTLED_POLLS_BEFORE_EXPIRY: row.status = "completed" raise _A_REJECTED_KEY @@ -4795,7 +4818,7 @@ async def test_expiring_a_batch_past_the_staleness_cutoff_keeps_a_finished_statu poller: Final = TestMultiPodBatchCostClaim._instance(TestMultiPodBatchCostClaim._prisma(row, []), router) with TestMultiPodBatchCostClaim._billing_patches([]): - for _ in range(FAILED_RETRIEVES_BEFORE_EXPIRY): + for _ in range(UNSETTLED_POLLS_BEFORE_EXPIRY): await poller.check_batch_cost() assert row.status == "completed"