diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index d1a3fccaac5..ce0774d6b8c 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -552,19 +552,16 @@ async def _fetch_before_delete( ) -> InteractionsAPIResponse | None: try: return await fetch_interaction(context) - except Exception as e: # noqa: BLE001 # unfetchable pre-delete state settles by releasing the reservation + except Exception as e: # noqa: BLE001 # the caller decides what an unfetchable pre-delete state means verbose_logger.debug( - "Could not fetch background interaction %s before delete, releasing its reservation: %s", - context.interaction_id, - e, + "Could not fetch background interaction %s before its delete: %s", context.interaction_id, e ) return None async def _settle_before_delete( - context: BackgroundInteractionPollContext, fetch_interaction: FetchInteraction + context: BackgroundInteractionPollContext, response: InteractionsAPIResponse | None ) -> SettlementOutcome | None: - response: Final = await _fetch_before_delete(context, fetch_interaction) if not await _claim(context): return None if response is None: @@ -581,7 +578,7 @@ async def maybe_settle_background_interaction_before_delete( ) -> SettlementOutcome | None: entry: Final = _ACTIVE_POLLS.get(interaction_id) if entry is not None: - return await _settle_before_delete(entry.context, fetch_interaction) + return await _settle_before_delete(entry.context, await _fetch_before_delete(entry.context, fetch_interaction)) settlement_store: Final = store or _STORE.store pending: Final = await _pending(settlement_store, interaction_id) if pending is None: @@ -596,7 +593,15 @@ async def maybe_settle_background_interaction_before_delete( api_base=api_base if isinstance(api_base, str) else None, store=settlement_store, ) - return await _settle_before_delete(context, fetch_interaction) + response: Final = await _fetch_before_delete(context, fetch_interaction) + if response is None: + verbose_logger.debug( + "Leaving background interaction %s to the poll that created it: this process could not fetch it with " + "the delete's credentials, so the delete is about to fail the same way", + interaction_id, + ) + return None + return await _settle_before_delete(context, response) async def _unclaimed(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]: diff --git a/tests/test_litellm/interactions/test_background_cost_polling.py b/tests/test_litellm/interactions/test_background_cost_polling.py index 40d63090054..df80acfd7e6 100644 --- a/tests/test_litellm/interactions/test_background_cost_polling.py +++ b/tests/test_litellm/interactions/test_background_cost_polling.py @@ -695,6 +695,30 @@ async def test_delete_on_another_replica_releases_the_create_reservation(): assert settled_metadata["user_api_key_budget_reservation"]["finalized"] is True +@pytest.mark.asyncio +async def test_delete_on_another_replica_leaves_an_unfetchable_create_to_its_own_poll(): + """ + The settling replica fetches with the delete's credentials, never the + create's, so a fetch it cannot make (a key only the deployment carries) + says nothing about the interaction: the delete is about to fail the same + way, and the poll on the creating replica still owns the bill. + """ + store = InMemoryBackgroundSettlementStore() + logging_obj = _logging_obj(litellm_params={"metadata": _create_metadata()}) + await _create_on_a_replica_that_then_dies(logging_obj, store) + fetch, _ = _fetch_sequence(RuntimeError("Google API key is required")) + + outcome = await maybe_settle_background_interaction_before_delete( + interaction_id="interactions/bg-abc", delete_kwargs={}, fetch_interaction=fetch, store=store + ) + + assert outcome is None + assert await store.is_claimed("interactions/bg-abc") is False + poll_fetch, _ = _fetch_sequence(_response("completed", with_usage=True)) + await asyncio.wait_for(_register_poll(logging_obj, poll_fetch=poll_fetch, store=store), timeout=5) + assert logging_obj.model_call_details["response_cost"] > 0 + + @pytest.mark.asyncio async def test_delete_settles_once_however_many_replicas_try(): store = InMemoryBackgroundSettlementStore()