From d55efe1e882d594c4c416238d440dc226b688360 Mon Sep 17 00:00:00 2001 From: Charan Rathore Date: Mon, 28 Sep 2026 11:07:04 +0530 Subject: [PATCH] test(proxy): run spend-log failure cases in CI shard --- .../test_proxy_update_spend.py | 94 ------------------- tests/unit/proxy/test_update_spend.py | 69 ++++++++++++++ 2 files changed, 69 insertions(+), 94 deletions(-) diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py index 9e1ec346b51..7099101db1c 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py @@ -917,97 +917,3 @@ async def test_update_spend_logs_parks_failed_batch_in_redis_with_wire_safe_date parked = await buffer.get_spend_logs_from_redis_buffer(limit=10) assert mock_prisma_client.spend_log_transactions == [] assert [(row["request_id"], row["startTime"]) for row in parked] == [("a", started.isoformat())] - - -@pytest.mark.asyncio -async def test_spend_log_serialization_failure_requeues_only_unwritten_tail( - mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.delenv("SPEND_LOGS_URL", raising=False) - rows = [make_spend_log_row(request_id=f"committed-{i}") for i in range(1000)] - rows.append(make_spend_log_row(request_id="unwritten")) - original_jsonify = mock_prisma_client.jsonify_object - - def jsonify(row: Any) -> Any: - if row["request_id"] == "unwritten": - raise TypeError("bad local serialization") - return original_jsonify(row) - - mock_prisma_client.jsonify_object = jsonify - mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock() - proxy_logging = MagicMock() - proxy_logging.failure_handler = AsyncMock() - with pytest.raises(TypeError, match="bad local serialization"): - await ProxyUpdateSpend.update_spend_logs( - n_retry_times=0, - prisma_client=mock_prisma_client, - db_writer_client=None, - proxy_logging_obj=proxy_logging, - logs_to_process=rows, - ) - assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count >= 1 - assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["unwritten"] - - -@pytest.mark.asyncio -async def test_external_spend_log_preflight_failure_requeues_without_post( - mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("SPEND_LOGS_URL", "http://writer.invalid") - rows = [make_spend_log_row(request_id="unwritten")] - rows[0]["unserializable"] = object() - writer = MagicMock() - writer.post = AsyncMock() - proxy_logging = MagicMock() - proxy_logging.failure_handler = AsyncMock() - with pytest.raises(TypeError): - await ProxyUpdateSpend.update_spend_logs( - n_retry_times=0, - prisma_client=mock_prisma_client, - db_writer_client=writer, - proxy_logging_obj=proxy_logging, - logs_to_process=rows, - ) - writer.post.assert_not_awaited() - assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["unwritten"] - - -@pytest.mark.asyncio -async def test_external_spend_log_post_error_is_not_replayed( - mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("SPEND_LOGS_URL", "http://writer.invalid") - writer = MagicMock() - writer.post = AsyncMock(side_effect=ValueError("uncertain delivery")) - proxy_logging = MagicMock() - proxy_logging.failure_handler = AsyncMock() - with pytest.raises(ValueError, match="uncertain delivery"): - await ProxyUpdateSpend.update_spend_logs( - n_retry_times=0, - prisma_client=mock_prisma_client, - db_writer_client=writer, - proxy_logging_obj=proxy_logging, - logs_to_process=[make_spend_log_row(request_id="maybe-committed")], - ) - writer.post.assert_awaited_once() - assert mock_prisma_client.spend_log_transactions == [] - - -@pytest.mark.asyncio -async def test_external_spend_log_transport_error_is_not_retried_or_requeued( - mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - import httpx - - monkeypatch.setenv("SPEND_LOGS_URL", "http://writer.invalid") - writer = MagicMock() - writer.post = AsyncMock(side_effect=httpx.ReadError("response lost after send")) - proxy_logging = MagicMock() - proxy_logging.failure_handler = AsyncMock() - with pytest.raises(httpx.ReadError): - await ProxyUpdateSpend.update_spend_logs( - n_retry_times=2, prisma_client=mock_prisma_client, db_writer_client=writer, - proxy_logging_obj=proxy_logging, logs_to_process=[make_spend_log_row(request_id="maybe-committed")], - ) - writer.post.assert_awaited_once() - assert mock_prisma_client.spend_log_transactions == [] diff --git a/tests/unit/proxy/test_update_spend.py b/tests/unit/proxy/test_update_spend.py index ebe505b3d60..39b285be21d 100644 --- a/tests/unit/proxy/test_update_spend.py +++ b/tests/unit/proxy/test_update_spend.py @@ -321,3 +321,72 @@ async def test_update_spend_logs_multiple_batches_with_failure(): # Verify all logs were cleared from transactions assert len(prisma_client.spend_log_transactions) == 0 + + +# These tests live in the proxy-db-db-and-spend CI shard, unlike the separate +# prisma_and_spend suite, so their branch coverage contributes to codecov. +def _spend_log_lifetime_fakes(): + client = MockPrismaClient() + logging = create_mock_proxy_logging() + logging.db_spend_update_writer.redis_update_buffer.store_spend_logs_in_redis = AsyncMock(return_value=False) + return client, logging + + +@pytest.mark.asyncio +async def test_spend_log_pre_write_failure_requeues_only_unwritten_tail(monkeypatch): + from litellm.proxy.utils import ProxyUpdateSpend + + monkeypatch.delenv("SPEND_LOGS_URL", raising=False) + client, logging = _spend_log_lifetime_fakes() + rows = [{"request_id": f"committed-{i}"} for i in range(1000)] + rows.append({"request_id": "unwritten"}) + + def jsonify(row): + if row["request_id"] == "unwritten": + raise TypeError("local conversion failed") + return row + + client.jsonify_object = jsonify + with pytest.raises(TypeError, match="local conversion failed"): + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=0, prisma_client=client, db_writer_client=None, + proxy_logging_obj=logging, logs_to_process=rows, + ) + assert client.db.litellm_spendlogs.create_many.await_count >= 1 + assert client.spend_log_transactions == [{"request_id": "unwritten"}] + + +@pytest.mark.asyncio +async def test_spend_log_external_preflight_requeues_without_post(monkeypatch): + from litellm.proxy.utils import ProxyUpdateSpend + + monkeypatch.setenv("SPEND_LOGS_URL", "http://writer.invalid") + client, logging = _spend_log_lifetime_fakes() + rows = [{"request_id": "unwritten", "bad": object()}] + writer = MagicMock() + writer.post = AsyncMock() + with pytest.raises(TypeError): + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=0, prisma_client=client, db_writer_client=writer, + proxy_logging_obj=logging, logs_to_process=rows, + ) + writer.post.assert_not_awaited() + assert client.spend_log_transactions == rows + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [ValueError("uncertain delivery"), httpx.ReadError("response lost after send")]) +async def test_spend_log_external_post_failure_never_retries_or_requeues(monkeypatch, failure): + from litellm.proxy.utils import ProxyUpdateSpend + + monkeypatch.setenv("SPEND_LOGS_URL", "http://writer.invalid") + client, logging = _spend_log_lifetime_fakes() + writer = MagicMock() + writer.post = AsyncMock(side_effect=failure) + with pytest.raises(type(failure)): + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=2, prisma_client=client, db_writer_client=writer, + proxy_logging_obj=logging, logs_to_process=[{"request_id": "maybe-committed"}], + ) + writer.post.assert_awaited_once() + assert client.spend_log_transactions == []