scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(lens): release budget reservations when the analysis model call fails (#44431)

Browse source 
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-03 14:39:27 -07:00 • committed by GitHub
parent 41a3781d4e
commit 98337c9334
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/proxy/lens/inference.py
View file

@ -192,7 +192,7 @@ async def analyze(
current, active.model_copy(update=MappingProxyType({"cost": active.cost + estimate}))
).model_copy(update=MappingProxyType({"spent": current.spent + estimate}))
def settle(e: Lens, cost: float) -> Lens:
def settle(e: Lens, cost: float, step: Step | None) -> Lens:
charged: Final = next((j for j in e.jobs if j.id == job.id), None)
adjusted: Final = (
e.model_copy(update=MappingProxyType({"spent": max(0, e.spent - estimate + cost)}))
@ -201,13 +201,8 @@ async def analyze(
)
if charged is None:
return adjusted
return replace_job(
adjusted,
add_step(
charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)})),
model_step(response, body, job.settings.model, cost),
),
)
refunded: Final = charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)}))
return replace_job(adjusted, add_step(refunded, step) if step is not None else refunded)
@asynccontextmanager
async def reserve_budget() -> AsyncIterator[None]:
@ -216,7 +211,7 @@ async def analyze(
try:
yield
except BaseException:
await repo.update(lens.id, lambda e: settle(e, 0))
await repo.update(lens.id, lambda e: settle(e, 0, None))
raise
data: Final[dict[str, object]] = { # mutable-ok: proxy processing enriches request data
@ -243,7 +238,8 @@ async def analyze(
response, billed_cost = await complete(worker.analysis_key_id, data, reserve_budget, request)
cost: Final = billed_cost if billed_cost is not None else completion_charge(deployments, response, estimate)
await repo.update(lens.id, lambda e: settle(e, cost))
step: Final = model_step(response, body, job.settings.model, cost)
await repo.update(lens.id, lambda e: settle(e, cost, step))
parsed: Final = Completion.model_validate_json(response.model_dump_json())
choice: Final = parsed.choices[0]
return ModelResult(