scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 174

fix(alerts): retry failed email delivery without replaying cached errors

Browse source
This commit is contained in:
moe-berri 2026-11-08 13:22:22 +08:00
parent d107312230
commit ed4623cd83
2 changed files with 19 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/integrations/SlackAlerting/usage_alerts.py
View file

@ -117,7 +117,7 @@ class UsageAlertDelivery:
if previous is not None:
if not previous.done():
raise RuntimeError("An alert email is still in flight; retry later")
if cls._email_key == key:
if cls._email_key == key and not previous.cancelled() and previous.exception() is None:
cls._email_task = None
cls._email_key = None
return previous.result()
@ -125,10 +125,12 @@ class UsageAlertDelivery:
cls._email_task = task
cls._email_key = key
task.add_done_callback(cls._email_finished)
await asyncio.shield(task)
if cls._email_task is task:
cls._email_task = None
cls._email_key = None
try:
await asyncio.shield(task)
finally:
if task.done() and cls._email_task is task:
cls._email_task = None
cls._email_key = None
@staticmethod
async def _send_email(recipient: str, subject: str, html: str) -> None:

16
tests/integration/observability/test_usage_alert_email.py
View file

@ -139,7 +139,10 @@ async def test_cancelled_smtp_attempt_retains_only_one_send(monkeypatch: pytest.
@pytest.mark.asyncio
async def test_smtp_greeting_over_two_seconds_still_delivers(monkeypatch: pytest.MonkeyPatch) -> None:
@pytest.mark.parametrize("fail_first", (False, True))
async def test_smtp_greeting_over_two_seconds_still_delivers(
monkeypatch: pytest.MonkeyPatch, fail_first: bool
) -> None:
messages: Final[list[bytes]] = []
async def slow_greeting(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
@ -152,7 +155,7 @@ async def test_smtp_greeting_over_two_seconds_still_delivers(monkeypatch: pytest
writer.write(b"354 End with a dot\r\n")
await writer.drain()
messages.append(await reader.readuntil(b"\r\n.\r\n"))
writer.write(b"250 Accepted\r\n")
writer.write(b"451 Retry later\r\n" if fail_first and len(messages) == 1 else b"250 Accepted\r\n")
elif command.upper().startswith(b"QUIT"):
writer.write(b"221 Closing\r\n")
break
@ -179,8 +182,13 @@ async def test_smtp_greeting_over_two_seconds_still_delivers(monkeypatch: pytest
}
)
try:
assert await UsageAlertDelivery(SlackAlerting(alerting=["email"], alert_types=[])).send_issue(pending)
assert len(messages) == 1 and b"View alert" in messages[0] and b"View source activity" in messages[0]
delivery: Final = UsageAlertDelivery(SlackAlerting(alerting=["email"], alert_types=[]))
if fail_first:
with pytest.raises(smtplib.SMTPDataError):
await delivery.send_issue(pending)
assert await delivery.send_issue(pending)
assert len(messages) == (2 if fail_first else 1)
assert b"View alert" in messages[-1] and b"View source activity" in messages[-1]
finally:
server.close()
await server.wait_closed()