fix(batches): preserve spend across malformed output and metadata failures (#47184)

Backport of #47184 to stable/1.104.x.
Cherry-picked from 3fb95c8fe3 (main).
Ported by hand onto the line's poller, whose surrounding code predates several main-only batch features: the claim now writes the terminal status and NUL-stripped file_object, a failed spend-log write restores both, and the managed-files postgres_storable_json sanitizer (from #45703 on main) is added to the line since this fix needs it.
This commit is contained in:
joshua-berri 2026-10-20 15:56:21 -07:00 • committed by Mrinal Chanshetty
parent bb1e40c3e5
commit f7b0b09e75
6 changed files with 230 additions and 117 deletions

View file

@ -2,6 +2,7 @@
Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked. Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
""" """
import json
from collections.abc import Sequence from collections.abc import Sequence
from dataclasses import replace as dataclasses_replace from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
@ -336,7 +337,7 @@ class CheckBatchCost:
"so it will no longer be polled" "so it will no longer be polled"
) )
async def _claim_job_for_costing(self, job: "_ManagedObjectRow") -> bool: async def _claim_job_for_costing(self, job: "_ManagedObjectRow", response: "LiteLLMBatch") -> bool:
""" """
Atomically flip batch_processed from false to true, returning whether this pod won Atomically flip batch_processed from false to true, returning whether this pod won
the row. Every pod and uvicorn worker schedules its own poller against the shared the row. Every pod and uvicorn worker schedules its own poller against the shared
@ -350,10 +351,16 @@ class CheckBatchCost:
""" """
if not self._has_batch_processed_column: if not self._has_batch_processed_column:
return True return True
from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
try: try:
claimed: Final = await _managed_object_table(self.prisma_client).update_many( claimed: Final = await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": False}, where={"id": job.id, "batch_processed": False},
data={"batch_processed": True}, data={
"batch_processed": True,
"status": response.status if response.status != "completed" else "complete",
"file_object": postgres_storable_json(response),
},
) )
except Exception as db_err: except Exception as db_err:
verbose_proxy_logger.error( verbose_proxy_logger.error(
@ -362,7 +369,9 @@ class CheckBatchCost:
return False return False
return claimed > 0 return claimed > 0
async def _release_job_claim(self, job: "_ManagedObjectRow") -> None: async def _release_job_claim(
self, job: "_ManagedObjectRow", original_status: str | None, original_file_object: object
) -> None:
"""Give a claimed row back once billing it failed, so a later poll cycle retries it. """Give a claimed row back once billing it failed, so a later poll cycle retries it.
Safe to match on batch_processed=True: while this poller is active the retrieve Safe to match on batch_processed=True: while this poller is active the retrieve
@ -374,7 +383,11 @@ class CheckBatchCost:
try: try:
await _managed_object_table(self.prisma_client).update_many( await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": True}, where={"id": job.id, "batch_processed": True},
data={"batch_processed": False}, data={
"batch_processed": False,
"status": original_status,
"file_object": json.dumps(original_file_object),
},
) )
except Exception as db_err: except Exception as db_err:
verbose_proxy_logger.error( verbose_proxy_logger.error(
@ -671,8 +684,6 @@ class CheckBatchCost:
@staticmethod @staticmethod
def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]: def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]:
import json
from litellm.types.utils import LiteLLMBatch from litellm.types.utils import LiteLLMBatch
file_object = job.file_object file_object = job.file_object
@ -899,7 +910,9 @@ class CheckBatchCost:
custom_llm_provider=str(llm_provider) if llm_provider else None, custom_llm_provider=str(llm_provider) if llm_provider else None,
) )
if not await self._claim_job_for_costing(job): original_status: Final = job.status
original_file_object: Final = job.file_object
if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
verbose_proxy_logger.info( verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod " f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
"in this window, so its cost is already being tracked there" "in this window, so its cost is already being tracked there"
@ -918,7 +931,7 @@ class CheckBatchCost:
batch_completion_cost=batch_result.completion_cost, batch_completion_cost=batch_result.completion_cost,
) )
except Exception: except Exception:
await self._release_job_claim(job) await self._release_job_claim(job, original_status, original_file_object)
raise raise
# Record batch duration (completed_at - created_at) # Record batch duration (completed_at - created_at)
@ -1061,22 +1074,19 @@ class CheckBatchCost:
# Track this job for the final metrics summary # Track this job for the final metrics summary
processed_models.append(tracked) processed_models.append(tracked)
# mark the job as complete if not self._has_batch_processed_column:
try: try:
update_data: dict = { await _managed_object_table(self.prisma_client).update(
"status": response.status if response.status != "completed" else "complete", where={"id": job.id},
"file_object": response.model_dump_json(), data={
} "status": response.status if response.status != "completed" else "complete",
if self._has_batch_processed_column: "file_object": response.model_dump_json(),
update_data["batch_processed"] = True },
await _managed_object_table(self.prisma_client).update( )
where={"id": job.id}, except Exception as db_err:
data=update_data, verbose_proxy_logger.error(
) f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
except Exception as db_err: )
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES: elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
from litellm.proxy.openai_files_endpoints.common_utils import ( from litellm.proxy.openai_files_endpoints.common_utils import (

View file

@ -22,7 +22,7 @@ from uuid import NAMESPACE_URL, uuid5
import httpx import httpx
from fastapi import HTTPException from fastapi import HTTPException
from pydantic import ValidationError from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
import litellm import litellm
from litellm import Router, verbose_logger from litellm import Router, verbose_logger
@ -205,6 +205,31 @@ def _managed_object_table(prisma_client: PrismaClient) -> _ManagedObjectTableAct
return prisma_client.db.litellm_managedobjecttable return prisma_client.db.litellm_managedobjecttable
_JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_POSTGRES_UNSTORABLE_CHARACTER: Final = "\x00"
def _storable_text(text: str) -> str:
return text.replace(_POSTGRES_UNSTORABLE_CHARACTER, "")
def _storable_json_value(value: JsonValue) -> JsonValue:
if isinstance(value, str):
return _storable_text(value)
if isinstance(value, list):
return [_storable_json_value(item) for item in value]
if isinstance(value, dict):
return {_storable_text(key): _storable_json_value(item) for key, item in value.items()}
return value
def postgres_storable_json(file_object: BaseModel) -> str:
dumped: Final = file_object.model_dump_json()
if "\\u0000" not in dumped:
return dumped
return _JSON_VALUE_ADAPTER.dump_json(_storable_json_value(_JSON_VALUE_ADAPTER.validate_json(dumped))).decode()
def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, str]: def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, str]:
hidden_params: Final = cast( # cast-ok: _hidden_params is an untyped attribute the upload path sets hidden_params: Final = cast( # cast-ok: _hidden_params is an untyped attribute the upload path sets
"Mapping[str, object]", getattr(file_object, "_hidden_params", None) or {} "Mapping[str, object]", getattr(file_object, "_hidden_params", None) or {}

View file

@ -3,7 +3,7 @@ from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass from dataclasses import dataclass
from dataclasses import replace as dataclasses_replace from dataclasses import replace as dataclasses_replace
from enum import Enum from enum import Enum
from typing import Any, Final, Literal from typing import Any, Final, Literal, cast
import litellm import litellm
from litellm._logging import verbose_logger from litellm._logging import verbose_logger
@ -744,9 +744,17 @@ def _batch_response_was_successful(
message batch results lines report ``result.type == "succeeded"``; Bedrock message batch results lines report ``result.type == "succeeded"``; Bedrock
batch output lines report ``modelOutput`` (and no ``error``). batch output lines report ``modelOutput`` (and no ``error``).
""" """
if custom_llm_provider == "anthropic":
return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
if custom_llm_provider == "bedrock": if custom_llm_provider == "bedrock":
return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
_response: Final[dict] = batch_job_output_file.get("response", None) or {} envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
return _response.get("status_code", None) == 200 envelope: Final = batch_job_output_file.get(envelope_key)
if not isinstance(envelope, Mapping):
if envelope is not None:
verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
return False
response: Final = cast(Mapping[object, object], envelope) # cast-ok: mapping shape validated above
return (
response.get("type") == "succeeded"
if custom_llm_provider == "anthropic"
else response.get("status_code") == 200
)

View file

@ -17,6 +17,7 @@ deterministic stand-ins so the arithmetic under test is the only variable.
import json import json
import logging import logging
from types import MappingProxyType from types import MappingProxyType
from typing import Final, Literal
import httpx import httpx
import pytest import pytest
@ -2417,3 +2418,30 @@ async def test_flag_sends_every_vertex_row_down_the_native_path_when_a_model_is_
assert calls == [] assert calls == []
assert (result.successful_requests, result.failed_requests) == (0, 1) assert (result.successful_requests, result.failed_requests) == (0, 1)
@pytest.mark.parametrize("provider", ("openai", "anthropic"))
@pytest.mark.parametrize("envelope", (1, "invalid", [1], None, {}))
@pytest.mark.parametrize("include_valid", (True, False))
def test_invalid_batch_envelopes_preserve_recoverable_cost(
provider: Literal["openai", "anthropic"], envelope: object, include_valid: bool
) -> None:
invalid: Final[dict[str, object]] = {"response" if provider == "openai" else "result": envelope}
valid: Final[dict[str, object]] = (
_success_row(model="batch-accounting-control", usage=_usage(100, 20))
if provider == "openai"
else {
"result": {
"type": "succeeded",
"message": {"model": "batch-accounting-control", "usage": {"input_tokens": 100, "output_tokens": 20}},
}
}
)
entries: Final = (valid, invalid, valid) if include_valid else (invalid,)
result: Final = bu._aggregate_batch_cost_usage_models(
entries, provider, model_info={"input_cost_per_token_batches": 0.001, "output_cost_per_token_batches": 0.002}
)
assert result.cost == pytest.approx(0.28 if include_valid else 0)
assert result.usage.prompt_tokens == (200 if include_valid else 0)
assert result.usage.completion_tokens == (40 if include_valid else 0)
assert (result.successful_requests, result.failed_requests) == (2 if include_valid else 0, 1)

View file

@ -9,6 +9,8 @@ with deployment credentials, bypassing the managed files access-control hooks.
""" """
import base64 import base64
import json
from typing import Final
import pytest import pytest
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch from unittest.mock import AsyncMock, MagicMock, patch
@ -367,13 +369,14 @@ async def test_check_batch_cost_should_call_afile_content_directly_with_credenti
# managed_files_obj.afile_content should NOT have been called # managed_files_obj.afile_content should NOT have been called
mock_managed_files_hook.afile_content.assert_not_called() mock_managed_files_hook.afile_content.assert_not_called()
# Verify the DB update writes batch_processed, status, and file_object claim_data: Final = next(
mock_prisma.db.litellm_managedobjecttable.update.assert_called_once() call.kwargs["data"]
update_call_kwargs = ( for call in mock_prisma.db.litellm_managedobjecttable.update_many.call_args_list
mock_prisma.db.litellm_managedobjecttable.update.call_args.kwargs if call.kwargs["where"].get("id") == mock_job.id
) )
assert update_call_kwargs["data"]["batch_processed"] is True assert claim_data["batch_processed"] is True
assert update_call_kwargs["data"]["status"] == "complete" assert claim_data["status"] == "complete"
assert ( persisted_batch: Final = json.loads(claim_data["file_object"])
"file_object" in update_call_kwargs["data"] assert persisted_batch["status"] == "completed"
), "file_object must be written to DB so list_batches reads updated status" assert persisted_batch["usage"]["prompt_tokens"] == 10
assert persisted_batch["usage"]["completion_tokens"] == 5

View file

@ -16,6 +16,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest import pytest
from fastapi import HTTPException from fastapi import HTTPException
from litellm.types.utils import LiteLLMBatch
if TYPE_CHECKING: if TYPE_CHECKING:
from litellm.batches.batch_utils import BatchCostUsageResult from litellm.batches.batch_utils import BatchCostUsageResult
@ -697,11 +699,15 @@ class TestCheckBatchCost:
return_value=[mock_job] return_value=[mock_job]
) )
mock_response = MagicMock() mock_response: Final = LiteLLMBatch(
mock_response.status = "completed" id="batch-1",
mock_response.output_file_id = "file-output-123" status="completed",
mock_response.model_dump_json.return_value = ( output_file_id="file-output-123",
'{"id":"batch-1","status":"completed"}' created_at=1,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
) )
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response) mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
@ -765,15 +771,13 @@ class TestCheckBatchCost:
await check_batch_cost_instance.check_batch_cost() await check_batch_cost_instance.check_batch_cost()
assert ( assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count >= 3, (
mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1 "Expected update() to be called exactly once for the completed job"
), "Expected update() to be called exactly once for the completed job" )
update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[ update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
1 assert update_data["batch_processed"] is True, (
]["data"] "update() must include batch_processed=True when column is present"
assert ( )
update_data["batch_processed"] is True
), "update() must include batch_processed=True when column is present"
assert update_data["status"] == "complete" assert update_data["status"] == "complete"
@pytest.mark.asyncio @pytest.mark.asyncio
@ -900,7 +904,7 @@ class TestCheckBatchCost:
mock_update_database.assert_awaited_once() mock_update_database.assert_awaited_once()
assert mock_update_database.call_args.kwargs["response_cost"] == 0.01 assert mock_update_database.call_args.kwargs["response_cost"] == 0.01
assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1, ( assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count >= 3, (
"the job must still be marked processed once cost tracking succeeds" "the job must still be marked processed once cost tracking succeeds"
) )
@ -1375,11 +1379,15 @@ class TestCheckBatchCost:
return_value=[mock_job] return_value=[mock_job]
) )
mock_response = MagicMock() mock_response: Final = LiteLLMBatch(
mock_response.status = terminal_status id="batch-1",
mock_response.output_file_id = "file-output-123" status=terminal_status,
mock_response.model_dump_json.return_value = ( output_file_id="file-output-123",
f'{{"id":"batch-1","status":"{terminal_status}"}}' created_at=1,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
) )
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response) mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
@ -1447,16 +1455,12 @@ class TestCheckBatchCost:
mock_afile_content.await_count == 1 mock_afile_content.await_count == 1
), f"{terminal_status} batch with an output file must fetch results and be billed" ), f"{terminal_status} batch with an output file must fetch results and be billed"
mock_logging_obj.async_success_handler.assert_awaited_once() mock_logging_obj.async_success_handler.assert_awaited_once()
assert ( assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count >= 3
mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1 update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[
1
]["data"]
assert update_data["batch_processed"] is True assert update_data["batch_processed"] is True
assert ( assert update_data["status"] == terminal_status, (
update_data["status"] == terminal_status f"billed {terminal_status} batch must keep its real terminal status in the DB"
), f"billed {terminal_status} batch must keep its real terminal status in the DB" )
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_error_file_failures_add_to_failed_request_count( async def test_error_file_failures_add_to_failed_request_count(
@ -1971,14 +1975,15 @@ class TestUnmanagedVertexRouting:
router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash" router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
router.get_model_ids.return_value = ["deploy-1"] router.get_model_ids.return_value = ["deploy-1"]
mock_response = MagicMock() mock_response: Final = LiteLLMBatch(
mock_response.status = "completed" id="8823717160934178816",
mock_response.output_file_id = "gs://bucket/out/predictions.jsonl" status="completed",
mock_response.error_file_id = None output_file_id="gs://bucket/out/predictions.jsonl",
mock_response.completed_at = None created_at=1,
mock_response.created_at = None completion_window="24h",
mock_response.model_dump_json.return_value = ( endpoint="/v1/chat/completions",
'{"id":"8823717160934178816","status":"completed"}' input_file_id="file-input-1",
object="batch",
) )
router.aretrieve_batch = AsyncMock(return_value=mock_response) router.aretrieve_batch = AsyncMock(return_value=mock_response)
router.get_deployment_credentials_with_provider = MagicMock( router.get_deployment_credentials_with_provider = MagicMock(
@ -2051,8 +2056,8 @@ class TestUnmanagedVertexRouting:
mock_logging_obj.async_success_handler.assert_awaited_once() mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01 assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01
assert prisma.db.litellm_managedobjecttable.update.call_count == 1 assert prisma.db.litellm_managedobjecttable.update_many.call_count >= 3
update_data = prisma.db.litellm_managedobjecttable.update.call_args[1]["data"] update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True assert update_data["batch_processed"] is True
assert update_data["status"] == "complete" assert update_data["status"] == "complete"
@ -2203,14 +2208,15 @@ class TestUnmanagedBedrockRouting:
router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4" router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4"
router.get_model_ids.return_value = ["deploy-1"] router.get_model_ids.return_value = ["deploy-1"]
mock_response = MagicMock() mock_response: Final = LiteLLMBatch(
mock_response.status = "completed" id=self._ARN,
mock_response.output_file_id = "s3://bucket/out/predictions.jsonl" status="completed",
mock_response.error_file_id = None output_file_id="s3://bucket/out/predictions.jsonl",
mock_response.completed_at = None created_at=1,
mock_response.created_at = None completion_window="24h",
mock_response.model_dump_json.return_value = ( endpoint="/v1/chat/completions",
f'{{"id":"{self._ARN}","status":"completed"}}' input_file_id="file-input-1",
object="batch",
) )
router.aretrieve_batch = AsyncMock(return_value=mock_response) router.aretrieve_batch = AsyncMock(return_value=mock_response)
router.get_deployment_credentials_with_provider = MagicMock( router.get_deployment_credentials_with_provider = MagicMock(
@ -2281,8 +2287,8 @@ class TestUnmanagedBedrockRouting:
mock_logging_obj.async_success_handler.assert_awaited_once() mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02 assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02
assert prisma.db.litellm_managedobjecttable.update.call_count == 1 assert prisma.db.litellm_managedobjecttable.update_many.call_count >= 3
update_data = prisma.db.litellm_managedobjecttable.update.call_args[1]["data"] update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True assert update_data["batch_processed"] is True
assert update_data["status"] == "complete" assert update_data["status"] == "complete"
@ -3150,13 +3156,20 @@ class TestMultiPodBatchCostClaim:
@staticmethod @staticmethod
def _router(): def _router():
response = MagicMock() from litellm.types.utils import LiteLLMBatch
response.status = "completed"
response.output_file_id = _CLAIM_OUTPUT_FILE_ID response: Final = LiteLLMBatch(
response.error_file_id = None id="batch-456",
response.created_at = 1 status="completed",
response.completed_at = 2 output_file_id=_CLAIM_OUTPUT_FILE_ID,
response.model_dump_json.return_value = '{"id":"batch-456","status":"completed"}' error_file_id=None,
created_at=1,
completed_at=2,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
)
deployment = MagicMock() deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "openai" deployment.litellm_params.custom_llm_provider = "openai"
@ -3238,8 +3251,9 @@ class TestMultiPodBatchCostClaim:
("openai/batch-test-model", 2, 0, "batch-test-model"), ("openai/batch-test-model", 2, 0, "batch-test-model"),
), ),
) )
@pytest.mark.parametrize("failure_envelope", (None, 1, "invalid", [1]))
async def test_batch_accounting_preserves_models_and_mixed_results( async def test_batch_accounting_preserves_models_and_mixed_results(
self, deployment_model: str, successes: int, failures: int, expected_model: str self, deployment_model: str, successes: int, failures: int, expected_model: str, failure_envelope: object
) -> None: ) -> None:
import base64 import base64
from datetime import datetime from datetime import datetime
@ -3274,7 +3288,7 @@ class TestMultiPodBatchCostClaim:
}, },
"error": None, "error": None,
} }
failure: Final = {"response": None, "error": {"code": "batch_expired", "message": "expired"}} failure: Final = {"response": failure_envelope, "error": {"code": "batch_expired", "message": "expired"}}
rows: Final = (success,) * successes + (failure,) * failures rows: Final = (success,) * successes + (failure,) * failures
content: Final = b"\n".join(json.dumps(row).encode() for row in rows) content: Final = b"\n".join(json.dumps(row).encode() for row in rows)
@ -3366,6 +3380,30 @@ class TestMultiPodBatchCostClaim:
with patch("litellm.proxy.proxy_server.scheduler", scheduler): with patch("litellm.proxy.proxy_server.scheduler", scheduler):
await guard._check_file_deletion_allowed(file_id) await guard._check_file_deletion_allowed(file_id)
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ("completed", "failed", "expired", "cancelled"))
@pytest.mark.parametrize("metadata_text", ("before\x00after", r"\u0000"))
async def test_cost_claim_persists_terminal_metadata_without_a_later_write(
self, status: str, metadata_text: str
) -> None:
row: Final = _FakeManagedObjectRow()
journal: Final = []
prisma: Final = self._prisma(row, journal)
router: Final = self._router()
router.aretrieve_batch.return_value = router.aretrieve_batch.return_value.model_copy(
update={"status": status, "metadata": {"annotation": metadata_text}}
)
prisma.db.litellm_managedobjecttable.update.side_effect = RuntimeError("final write unavailable")
worker: Final = self._instance(prisma, router)
with self._billing_patches(journal) as logging_obj:
await worker.check_batch_cost()
await worker.check_batch_cost()
assert row.batch_processed is True
assert row.status == ("complete" if status == "completed" else status)
assert json.loads(row.file_object)["status"] == status
assert json.loads(row.file_object)["metadata"] == {"annotation": metadata_text.replace("\x00", "")}
assert logging_obj.async_success_handler.await_count == 1
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_winning_pod_claims_the_row_between_fetching_and_billing(self): async def test_winning_pod_claims_the_row_between_fetching_and_billing(self):
"""The claim flips batch_processed false -> true after the results are in hand and """The claim flips batch_processed false -> true after the results are in hand and
@ -3377,13 +3415,9 @@ class TestMultiPodBatchCostClaim:
with self._billing_patches(journal) as logging_obj: with self._billing_patches(journal) as logging_obj:
await self._instance(prisma, self._router()).check_batch_cost() await self._instance(prisma, self._router()).check_batch_cost()
assert journal == ["fetch", "claim", "bill", "finalize"] assert journal == ["fetch", "claim", "bill"]
assert self._claim_calls(prisma) == [ assert row.status == "complete"
{ assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5}
"where": {"id": "job-claim-1", "batch_processed": False},
"data": {"batch_processed": True},
}
]
logging_obj.async_success_handler.assert_awaited_once() logging_obj.async_success_handler.assert_awaited_once()
assert row.batch_processed is True assert row.batch_processed is True
@ -3403,18 +3437,17 @@ class TestMultiPodBatchCostClaim:
assert journal == ["fetch"] assert journal == ["fetch"]
logging_obj.async_success_handler.assert_not_awaited() logging_obj.async_success_handler.assert_not_awaited()
assert self._claim_calls(prisma) == [ assert row.status == "in_progress"
{ assert json.loads(row.file_object)["status"] == "in_progress"
"where": {"id": "job-claim-1", "batch_processed": False},
"data": {"batch_processed": True},
}
]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_a_failed_spend_log_write_releases_the_claim(self): @pytest.mark.parametrize("stored_as_json_string", (True, False))
async def test_a_failed_spend_log_write_releases_the_claim(self, stored_as_json_string: bool) -> None:
"""A transient failure while billing a claimed batch must hand the row back, or its """A transient failure while billing a claimed batch must hand the row back, or its
spend is silently lost instead of being retried on the next cycle.""" spend is silently lost instead of being retried on the next cycle."""
row = _FakeManagedObjectRow() row = _FakeManagedObjectRow()
original_file_object: Final = row.file_object if stored_as_json_string else json.loads(row.file_object)
row.file_object = original_file_object
journal = [] journal = []
prisma = self._prisma(row, journal) prisma = self._prisma(row, journal)
@ -3423,10 +3456,16 @@ class TestMultiPodBatchCostClaim:
assert journal == ["fetch", "claim", "bill", "release"] assert journal == ["fetch", "claim", "bill", "release"]
assert row.batch_processed is False assert row.batch_processed is False
assert self._claim_calls(prisma)[-1] == { assert row.status == "in_progress"
"where": {"id": "job-claim-1", "batch_processed": True}, assert json.loads(row.file_object) == original_file_object
"data": {"batch_processed": False},
} with self._billing_patches(journal) as logging_obj:
await self._instance(prisma, self._router()).check_batch_cost()
await self._instance(prisma, self._router()).check_batch_cost()
assert row.batch_processed is True
assert row.status == "complete"
logging_obj.async_success_handler.assert_awaited_once()
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_a_worker_interrupted_mid_costing_leaves_the_batch_billable(self): async def test_a_worker_interrupted_mid_costing_leaves_the_batch_billable(self):
@ -3460,7 +3499,7 @@ class TestMultiPodBatchCostClaim:
with self._billing_patches(survivor_journal) as survivor_logging: with self._billing_patches(survivor_journal) as survivor_logging:
await self._instance(survivor_prisma, self._router()).check_batch_cost() await self._instance(survivor_prisma, self._router()).check_batch_cost()
assert survivor_journal == ["fetch", "claim", "bill", "finalize"] assert survivor_journal == ["fetch", "claim", "bill"]
survivor_logging.async_success_handler.assert_awaited_once() survivor_logging.async_success_handler.assert_awaited_once()
assert row.batch_processed is True assert row.batch_processed is True
@ -3493,7 +3532,7 @@ class TestMultiPodBatchCostClaim:
finish_fetch.set() finish_fetch.set()
await asyncio.wait_for(costing, timeout=5) await asyncio.wait_for(costing, timeout=5)
assert journal == ["fetch", "claim", "bill", "finalize"] assert journal == ["fetch", "claim", "bill"]
assert row.batch_processed is True assert row.batch_processed is True
await self._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID) await self._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)