mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-05 02:41:56 +00:00
fix(interactions): leave an unfetchable background interaction to its creating poll when a delete lands elsewhere
The remote pre-delete path fetches with only the delete's credentials, so a fetch it cannot make says nothing about the interaction. It used to claim the settlement row and release the reservation anyway, which stopped the creating replica's poll and lost the bill when the delete then failed the same way. It now returns without claiming; the in-process path keeps releasing on an unfetchable state, since its context carries the create's own credentials.
This commit is contained in:
parent
59c2ce4fe6
commit
ac68891ef2
2 changed files with 37 additions and 8 deletions
|
|
@ -552,19 +552,16 @@ async def _fetch_before_delete(
|
|||
) -> InteractionsAPIResponse | None:
|
||||
try:
|
||||
return await fetch_interaction(context)
|
||||
except Exception as e: # noqa: BLE001 # unfetchable pre-delete state settles by releasing the reservation
|
||||
except Exception as e: # noqa: BLE001 # the caller decides what an unfetchable pre-delete state means
|
||||
verbose_logger.debug(
|
||||
"Could not fetch background interaction %s before delete, releasing its reservation: %s",
|
||||
context.interaction_id,
|
||||
e,
|
||||
"Could not fetch background interaction %s before its delete: %s", context.interaction_id, e
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _settle_before_delete(
|
||||
context: BackgroundInteractionPollContext, fetch_interaction: FetchInteraction
|
||||
context: BackgroundInteractionPollContext, response: InteractionsAPIResponse | None
|
||||
) -> SettlementOutcome | None:
|
||||
response: Final = await _fetch_before_delete(context, fetch_interaction)
|
||||
if not await _claim(context):
|
||||
return None
|
||||
if response is None:
|
||||
|
|
@ -581,7 +578,7 @@ async def maybe_settle_background_interaction_before_delete(
|
|||
) -> SettlementOutcome | None:
|
||||
entry: Final = _ACTIVE_POLLS.get(interaction_id)
|
||||
if entry is not None:
|
||||
return await _settle_before_delete(entry.context, fetch_interaction)
|
||||
return await _settle_before_delete(entry.context, await _fetch_before_delete(entry.context, fetch_interaction))
|
||||
settlement_store: Final = store or _STORE.store
|
||||
pending: Final = await _pending(settlement_store, interaction_id)
|
||||
if pending is None:
|
||||
|
|
@ -596,7 +593,15 @@ async def maybe_settle_background_interaction_before_delete(
|
|||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
store=settlement_store,
|
||||
)
|
||||
return await _settle_before_delete(context, fetch_interaction)
|
||||
response: Final = await _fetch_before_delete(context, fetch_interaction)
|
||||
if response is None:
|
||||
verbose_logger.debug(
|
||||
"Leaving background interaction %s to the poll that created it: this process could not fetch it with "
|
||||
"the delete's credentials, so the delete is about to fail the same way",
|
||||
interaction_id,
|
||||
)
|
||||
return None
|
||||
return await _settle_before_delete(context, response)
|
||||
|
||||
|
||||
async def _unclaimed(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
|
||||
|
|
|
|||
|
|
@ -695,6 +695,30 @@ async def test_delete_on_another_replica_releases_the_create_reservation():
|
|||
assert settled_metadata["user_api_key_budget_reservation"]["finalized"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_on_another_replica_leaves_an_unfetchable_create_to_its_own_poll():
|
||||
"""
|
||||
The settling replica fetches with the delete's credentials, never the
|
||||
create's, so a fetch it cannot make (a key only the deployment carries)
|
||||
says nothing about the interaction: the delete is about to fail the same
|
||||
way, and the poll on the creating replica still owns the bill.
|
||||
"""
|
||||
store = InMemoryBackgroundSettlementStore()
|
||||
logging_obj = _logging_obj(litellm_params={"metadata": _create_metadata()})
|
||||
await _create_on_a_replica_that_then_dies(logging_obj, store)
|
||||
fetch, _ = _fetch_sequence(RuntimeError("Google API key is required"))
|
||||
|
||||
outcome = await maybe_settle_background_interaction_before_delete(
|
||||
interaction_id="interactions/bg-abc", delete_kwargs={}, fetch_interaction=fetch, store=store
|
||||
)
|
||||
|
||||
assert outcome is None
|
||||
assert await store.is_claimed("interactions/bg-abc") is False
|
||||
poll_fetch, _ = _fetch_sequence(_response("completed", with_usage=True))
|
||||
await asyncio.wait_for(_register_poll(logging_obj, poll_fetch=poll_fetch, store=store), timeout=5)
|
||||
assert logging_obj.model_call_details["response_cost"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_settles_once_however_many_replicas_try():
|
||||
store = InMemoryBackgroundSettlementStore()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue