From 923c813d47a5f7778056989154b4e62fa2fb1e77 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:31:27 -0700 Subject: [PATCH] fix(cost): preserve pass-through accounting across cancellation --- .../pass_through_endpoints/success_handler.py | 41 +++-- .../test_success_handler.py | 154 ++++++++++++++++++ 2 files changed, 183 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 38934422c71..bafdd9e197d 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -1,13 +1,17 @@ +import asyncio import json +from contextlib import suppress from datetime import datetime from types import MappingProxyType from typing import Any, Final from urllib.parse import urlparse +import anyio import httpx from litellm._logging import verbose_proxy_logger from litellm.constants import AZURE_SPEECH_CUSTOM_LLM_PROVIDER +from litellm.litellm_core_utils.core_helpers import bind_budget_reservation_to_callbacks from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import PassThroughEndpointLoggingResultValues from litellm.types.passthrough_endpoints.pass_through_endpoints import ( @@ -144,19 +148,32 @@ class PassThroughEndpointLogging: from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost - await finalize_proxy_response_cost(logging_obj, standard_logging_response_object) - GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - async_coroutine=self.handle_logging( - logging_obj=logging_obj, - standard_logging_response_object=standard_logging_response_object, - result=result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - finalize_before_dispatch=False, - **kwargs, + async def finalize_and_enqueue() -> None: + await finalize_proxy_response_cost(logging_obj, standard_logging_response_object) + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + async_coroutine=self.handle_logging( + logging_obj=logging_obj, + standard_logging_response_object=standard_logging_response_object, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + finalize_before_dispatch=False, + **kwargs, + ) ) - ) + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) + + pending: Final = asyncio.create_task(finalize_and_enqueue()) + try: + await asyncio.shield(pending) + except asyncio.CancelledError: + with anyio.CancelScope(shield=True): + while not pending.done(): + with suppress(asyncio.CancelledError): + await asyncio.shield(pending) + pending.result() + raise return await logging_obj.dispatch_success_handlers( diff --git a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py index 2cc03949ff1..c721e808ed5 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py @@ -648,3 +648,157 @@ def test_vertex_ai_live_route_without_usage_frames_yields_no_logging_response( assert normalized["standard_logging_response_object"] is None assert normalized["kwargs"] == {"model": _LIVE_MODEL} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_mode", ["once", "twice", "scope", "none"]) +@pytest.mark.parametrize("cancel_phase", ["allocation", "lock"]) +async def test_completed_passthrough_cancellation_preserves_accounting( + monkeypatch: pytest.MonkeyPatch, cancel_mode: str, cancel_phase: str +) -> None: + from unittest.mock import create_autospec + + import anyio + + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.spend_tracking.budget_reservation import release_unbound_budget_reservation + from litellm.proxy.utils import ProxyLogging + from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage + + class AllowanceCounter: + def __init__(self) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + self.calls = 0 + self.units = 0.0 + + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: + self.started.set() + await self.release.wait() + self.calls += 1 + self.units += value + return self.units + + now: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) + model: Final = "passthrough-cancellation-test" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "web_search_billing_unit": "per_query", + "search_context_cost_per_query": {"search_context_size_medium": 0.01}, + "search_free_requests": {"count": 2, "period": "month", "shared_group": "cancelled-pass-through"}, + }, + ) + monkeypatch.setattr(litellm, "apply_search_free_tier", True) + monkeypatch.setattr(proxy_server, "general_settings", {}) + counters: Final = DualCache() + for key in ("spend:key:pass-through-key", "spend:team:pass-through-team"): + counters.set_cache(key=key, value=1.9) + monkeypatch.setattr(proxy_server, "spend_counter_cache", counters) + reservation: Final = { + "reserved_cost": 0.4, + "callback_bound": False, + "entries": [ + {"counter_key": key, "entity_type": "Key", "entity_id": key, "reserved_cost": 0.4} + for key in ("spend:key:pass-through-key", "spend:team:pass-through-team") + ], + } + writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.db_spend_update_writer = writer + allowance: Final = AllowanceCounter() + monkeypatch.setattr(proxy_logging.internal_usage_cache.dual_cache, "redis_cache", allowance) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging) + logging_obj: Final = LiteLLMLoggingObj( + model=model, + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=now, + litellm_call_id="cancelled-pass-through", + function_id="cancelled-pass-through", + ) + logging_obj.add_dynamic_callback(ProxyDBLogger(spend_writer=lambda: writer)) + logging_obj.update_environment_variables( + model=model, + custom_llm_provider="vertex_ai", + optional_params={}, + litellm_params={ + "metadata": { + "user_api_key": "pass-through-key", + "user_api_key_user_id": "pass-through-user", + "user_api_key_team_id": "pass-through-team", + "user_api_key_budget_reservation": reservation, + } + }, + ) + result: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=3), + ), + ) + result._hidden_params["response_cost"] = 0.5 + scope: Final = anyio.CancelScope() + entered: Final = asyncio.Event() + + async def dispatch() -> None: + with scope: + entered.set() + await PassThroughEndpointLogging().handle_logging( + logging_obj=logging_obj, + standard_logging_response_object=result, + result="", + start_time=now, + end_time=now, + cache_hit=False, + finalize_before_dispatch=True, + ) + + if cancel_phase == "lock": + await logging_obj.cost_finalization_lock.acquire() + pending: Final = asyncio.create_task(dispatch()) + await entered.wait() + if cancel_phase == "allocation": + await allowance.started.wait() + if cancel_mode == "scope": + scope.cancel() + elif cancel_mode != "none": + pending.cancel() + await asyncio.sleep(0) + if cancel_mode == "twice": + pending.cancel() + if cancel_phase == "lock": + logging_obj.cost_finalization_lock.release() + allowance.release.set() + if cancel_mode in ("once", "twice"): + with pytest.raises(asyncio.CancelledError): + await pending + else: + await pending + assert scope.cancelled_caught is (cancel_mode == "scope") + assert reservation["callback_bound"] is True + await release_unbound_budget_reservation(reservation) + await GLOBAL_LOGGING_WORKER.flush() + assert allowance.calls == 1 + assert allowance.units == 3 + assert writer.update_database.await_count == 1 + recorded: Final = writer.update_database.call_args.kwargs + assert recorded["response_cost"] == pytest.approx(0.48) + payload: Final = recorded["kwargs"]["standard_logging_object"] + assert payload["cost_finalized"] is True + assert (payload["prompt_tokens"], payload["completion_tokens"]) == (10, 5) + assert reservation["finalized"] is True + for key in ("spend:key:pass-through-key", "spend:team:pass-through-team"): + assert await counters.async_get_cache(key=key) == pytest.approx(1.98)