diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 082552c58e1..a5a9e997677 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7282,6 +7282,7 @@ class ProxyUpdateSpend: start_time: Final = time.time() try: for i in range(n_retry_times + 1): + external_post_attempted = False try: base_url = os.getenv("SPEND_LOGS_URL", None) if len(logs_to_process) > 0 and base_url is not None and db_writer_client is not None: @@ -7294,6 +7295,7 @@ class ProxyUpdateSpend: # No external request has been sent. The batch is safe to replay. await requeue_spend_logs(prisma_client, proxy_logging_obj, logs_to_process) raise + external_post_attempted = True response = await db_writer_client.post( url=base_url + "spend/update", data=json_data, @@ -7336,6 +7338,10 @@ class ProxyUpdateSpend: ) break except Exception as e: + if external_post_attempted: + # Even a transport error can arrive after the remote writer + # committed. Retrying or requeueing could duplicate spend. + raise if not _is_transient_spend_log_write_error(e): if PrismaDBExceptionHandler.is_prisma_error(e): await requeue_spend_logs(prisma_client, proxy_logging_obj, logs_to_process) diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py index 89afe852427..9e1ec346b51 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py @@ -991,3 +991,23 @@ async def test_external_spend_log_post_error_is_not_replayed( ) writer.post.assert_awaited_once() assert mock_prisma_client.spend_log_transactions == [] + + +@pytest.mark.asyncio +async def test_external_spend_log_transport_error_is_not_retried_or_requeued( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import httpx + + monkeypatch.setenv("SPEND_LOGS_URL", "http://writer.invalid") + writer = MagicMock() + writer.post = AsyncMock(side_effect=httpx.ReadError("response lost after send")) + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + with pytest.raises(httpx.ReadError): + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=2, prisma_client=mock_prisma_client, db_writer_client=writer, + proxy_logging_obj=proxy_logging, logs_to_process=[make_spend_log_row(request_id="maybe-committed")], + ) + writer.post.assert_awaited_once() + assert mock_prisma_client.spend_log_transactions == []