mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(cleanup): scope dead-daemon update_many to successfully-terminated sessions only
Greptile P1 (review #PRR_kwDOKALCgc78v9wy):
The bulk update_many at the end of _sweep_dead_daemons used the full
'rows' list as the ID set, even for sessions where
_terminate_session_internal had thrown. The outer try/except swallowed
the error, but the row was still in 'ids' and got marked terminal
('error') without provider.terminate ever firing. Once Epic B wires
in a real EC2 provider, that's a silent VM/cost leak per failed sweep.
Track terminated_ids inside the try block instead — only successfully
terminated rows get the bulk status update. Failed rows stay
non-terminal so the next sweep pass retries them.
Also early-return when no rows succeed, so we don't fire an empty
update_many.
This commit is contained in:
parent
c71535dfe7
commit
b735e32ff6
1 changed files with 17 additions and 4 deletions
|
|
@ -92,9 +92,17 @@ async def _sweep_dead_daemons(prisma_client) -> int:
|
|||
# Terminate via the shared helper. It cancels active runs (with
|
||||
# `run_cancelled` events), flips the session status, AND fires
|
||||
# ``provider.terminate`` — exactly what we want here.
|
||||
#
|
||||
# We track ONLY sessions that completed termination cleanly. Failed
|
||||
# rows stay non-terminal so the next sweep pass can retry them —
|
||||
# without that, a transient DB error would mark the row ``error``
|
||||
# via the bulk update_many below and silently orphan its VM forever
|
||||
# (Greptile P1).
|
||||
terminated_ids: list = []
|
||||
for row in rows:
|
||||
try:
|
||||
await _terminate_session_internal(row.id, reason="daemon_dead")
|
||||
terminated_ids.append(row.id)
|
||||
except Exception as exc:
|
||||
verbose_proxy_logger.exception(
|
||||
"sweeper: failed to terminate dead-daemon session=%s: %s",
|
||||
|
|
@ -102,13 +110,17 @@ async def _sweep_dead_daemons(prisma_client) -> int:
|
|||
exc,
|
||||
)
|
||||
|
||||
if not terminated_ids:
|
||||
return 0
|
||||
|
||||
# Daemon-dead sessions land in ``error`` (not ``terminated``); the
|
||||
# shared helper sets ``terminated`` so we explicitly downgrade to
|
||||
# ``error`` here. (Both are terminal — no further state transitions.)
|
||||
# Scope the bulk update to ONLY successfully-terminated rows so a
|
||||
# mid-flight failure doesn't get marked terminal-without-VM-teardown.
|
||||
now = _now()
|
||||
ids = [r.id for r in rows]
|
||||
await prisma_client.db.litellm_agentsession.update_many(
|
||||
where={"id": {"in": ids}},
|
||||
where={"id": {"in": terminated_ids}},
|
||||
data={
|
||||
"status": SESSION_STATUS_ERROR,
|
||||
"updated_at": now,
|
||||
|
|
@ -116,9 +128,10 @@ async def _sweep_dead_daemons(prisma_client) -> int:
|
|||
)
|
||||
|
||||
verbose_proxy_logger.info(
|
||||
"sweeper: terminated %d sessions (dead daemon, via provider)", len(ids)
|
||||
"sweeper: terminated %d sessions (dead daemon, via provider)",
|
||||
len(terminated_ids),
|
||||
)
|
||||
return len(ids)
|
||||
return len(terminated_ids)
|
||||
|
||||
|
||||
async def _sweep_stuck_runs(prisma_client) -> int:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue