diff --git a/tests/integration/observability/test_user_soft_budget_alert.py b/tests/integration/observability/test_user_soft_budget_alert.py
index f76b2c0583a..c600254b24e 100644
--- a/tests/integration/observability/test_user_soft_budget_alert.py
+++ b/tests/integration/observability/test_user_soft_budget_alert.py
@@ -1,37 +1,133 @@
-from collections.abc import Iterator
+import asyncio
+import json
+import threading
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Final
+import httpx
import pytest
import yaml
+from anthropic import Anthropic, AsyncAnthropic
from integration._support.client import (
JSON_OBJECT,
Gateway,
+ Scenario,
eventually,
gateway_from_environment,
+ held,
object_value,
string_value,
)
-from integration._support.process import graceful_stop_seconds, owned_proxy
+from integration._support.mail import Mailbox, smtp_sink
+from integration._support.ports import allocate_port
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy, owned_proxy_process
+from integration._support.upstream import delete_scenario, register_scenario
from integration._support.wire import Reply, Request, Wire, wire_server
+from integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse
+from openai import AsyncOpenAI, OpenAI
+from pydantic import JsonValue
-pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 120), pytest.mark.shares_worker_state]
+pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 240), pytest.mark.shares_worker_state]
ALERT_SECONDS: Final = 40
+QUIET_SECONDS: Final = 3
+SOFT_BUDGET: Final = 0.5
+SPEND: Final = 1.0
+REPLY: Final = "soft budget reply"
+RESPONSE: Final[dict[str, JsonValue]] = {
+ "id": "resp_$UNIQUE_ID",
+ "object": "response",
+ "created_at": 1,
+ "status": "completed",
+ "model": "gpt-4o-mini",
+ "output": [
+ {
+ "id": "msg_$UNIQUE_ID",
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{"type": "output_text", "text": REPLY, "annotations": []}],
+ }
+ ],
+ "usage": {"input_tokens": 20, "output_tokens": 20, "total_tokens": 40},
+}
+RESPONSE_EVENTS: Final[tuple[Mapping[str, JsonValue], ...]] = (
+ {"type": "response.created", "sequence_number": 0, "response": {**RESPONSE, "status": "in_progress", "output": []}},
+ {
+ "type": "response.output_text.delta",
+ "sequence_number": 1,
+ "item_id": "msg_$UNIQUE_ID",
+ "output_index": 0,
+ "content_index": 0,
+ "delta": REPLY,
+ },
+ {"type": "response.completed", "sequence_number": 2, "response": RESPONSE},
+)
+
+
+class AlertLog:
+ """Every body the alert sink received so far; reading it never loses an earlier alert."""
+
+ def __init__(self, wire: Wire) -> None:
+ self.wire: Final = wire
+ self._bodies: tuple[str, ...] = ()
+
+ def bodies(self) -> tuple[str, ...]:
+ self._bodies = (*self._bodies, *(request.body.decode() for request in self.wire.drain()))
+ return self._bodies
+
+ def naming(self, marker: str) -> tuple[str, ...]:
+ return tuple(body for body in self.bodies() if marker in body)
+
+
+@dataclass(frozen=True, slots=True)
+class SinkControls:
+ refusing: threading.Event
+ stalling: threading.Event
+ release: threading.Event
@dataclass(frozen=True, slots=True)
class AlertingProxy:
gateway: Gateway
- alerts: Wire
+ alerts: AlertLog
+ sink: SinkControls
+
+
+@dataclass(frozen=True, slots=True)
+class MailingProxy:
+ gateway: Gateway
+ webhooks: AlertLog
+ mailbox: Mailbox
+
+
+@dataclass(frozen=True, slots=True)
+class ChaosProxy:
+ proxy: OwnedProxy
+ sink_port: int
+
+
+def alert_reply(controls: SinkControls) -> Callable[[Request], Reply]:
+ def respond(request: Request) -> Reply:
+ if controls.refusing.is_set():
+ return Reply(status=403, body=b"invalid_token", content_type="text/plain")
+ if controls.stalling.is_set():
+ return Reply(chunks=(b"ok",), content_type="text/plain", gate_before_first=controls.release)
+ return Reply(body=b"ok", content_type="text/plain")
+
+ return respond
def accept_alert(request: Request) -> Reply:
return Reply(body=b"ok", content_type="text/plain")
-def alerting_config(directory: Path) -> Path:
+def alerting_config(directory: Path, name: str, **general_settings: JsonValue) -> Path:
base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
config: Final = {
**base,
@@ -39,38 +135,321 @@ def alerting_config(directory: Path) -> Path:
**object_value(base["general_settings"]),
"alerting": ["slack"],
"alert_types": ["budget_alerts"],
+ **general_settings,
},
}
- path: Final = directory / "alerting.yaml"
+ path: Final = directory / f"{name}.yaml"
path.write_text(yaml.safe_dump(config))
return path
-@pytest.fixture(scope="module")
-def alerting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[AlertingProxy]:
- directory: Final = tmp_path_factory.mktemp("user-soft-budget-alerts")
- with gateway_from_environment() as shared, wire_server(accept_alert) as alerts:
+@contextmanager
+def slack_alerting(directory: Path, name: str, **general_settings: JsonValue) -> Iterator[AlertingProxy]:
+ controls: Final = SinkControls(threading.Event(), threading.Event(), threading.Event())
+ with gateway_from_environment() as shared, wire_server(alert_reply(controls)) as alerts:
with owned_proxy(
shared,
directory,
{"SLACK_WEBHOOK_URL": alerts.url, "DEFAULT_FLUSH_INTERVAL_SECONDS": "1"},
- config=alerting_config(directory),
+ config=alerting_config(directory, name, **general_settings),
) as gateway:
- yield AlertingProxy(gateway, alerts)
+ yield AlertingProxy(gateway, AlertLog(alerts), controls)
-def alerts_naming(alerts: Wire, marker: str, *, seconds: float) -> tuple[str, ...]:
- def matching() -> tuple[str, ...]:
- texts: Final = (string_value(JSON_OBJECT.validate_json(request.body)["text"]) for request in alerts.drain())
- return tuple(text for text in texts if marker in text)
-
- return eventually(matching, lambda found: len(found) > 0, seconds=seconds)
+@pytest.fixture(scope="module")
+def alerting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[AlertingProxy]:
+ with slack_alerting(tmp_path_factory.mktemp("user-soft-budget-alerts"), "alerting") as proxy:
+ yield proxy
-def user_info(gateway: Gateway, user_id: str) -> dict[str, object]:
+@pytest.fixture(scope="module")
+def team_key_alerting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[AlertingProxy]:
+ directory: Final = tmp_path_factory.mktemp("user-soft-budget-team-keys")
+ with slack_alerting(directory, "team-keys", apply_user_budget_to_team_keys=True) as proxy:
+ yield proxy
+
+
+@pytest.fixture(scope="module")
+def fail_closed_alerting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[AlertingProxy]:
+ directory: Final = tmp_path_factory.mktemp("user-soft-budget-fail-closed")
+ with slack_alerting(directory, "fail-closed", fail_closed_budget_enforcement=True) as proxy:
+ yield proxy
+
+
+@pytest.fixture(scope="module")
+def mailing(tmp_path_factory: pytest.TempPathFactory) -> Iterator[MailingProxy]:
+ directory: Final = tmp_path_factory.mktemp("user-soft-budget-mail")
+ with (
+ gateway_from_environment() as shared,
+ wire_server(accept_alert) as slack,
+ wire_server(accept_alert) as webhooks,
+ smtp_sink() as mailbox,
+ ):
+ overrides: Final = {
+ "SLACK_WEBHOOK_URL": slack.url,
+ "WEBHOOK_URL": webhooks.url,
+ "SMTP_HOST": mailbox.host,
+ "SMTP_PORT": str(mailbox.port),
+ "SMTP_TLS": "False",
+ "SMTP_SENDER_EMAIL": "alerts@integration.test",
+ }
+ config: Final = alerting_config(directory, "mailing", alerting=["slack", "webhook", "email"])
+ with owned_proxy(shared, directory, overrides, config=config) as gateway:
+ yield MailingProxy(gateway, AlertLog(webhooks), mailbox)
+
+
+@pytest.fixture(scope="module")
+def chaos(tmp_path_factory: pytest.TempPathFactory) -> Iterator[ChaosProxy]:
+ directory: Final = tmp_path_factory.mktemp("user-soft-budget-chaos")
+ sink_port: Final = allocate_port()
+ with gateway_from_environment() as shared:
+ with owned_proxy_process(
+ shared,
+ directory,
+ {"SLACK_WEBHOOK_URL": f"http://127.0.0.1:{sink_port}", "DEFAULT_FLUSH_INTERVAL_SECONDS": "1"},
+ config=alerting_config(directory, "chaos"),
+ workers=2,
+ ) as proxy:
+ yield ChaosProxy(proxy, sink_port)
+
+
+def slack_text(body: str) -> str:
+ return string_value(JSON_OBJECT.validate_json(body)["text"])
+
+
+def alerts_naming(alerts: AlertLog, marker: str, *, seconds: float = ALERT_SECONDS) -> tuple[str, ...]:
+ return eventually(lambda: alerts.naming(marker), lambda found: len(found) > 0, seconds=seconds)
+
+
+def user_info(gateway: Gateway, user_id: str) -> dict[str, JsonValue]:
return object_value(gateway.get("/user/info", {"user_id": user_id})["user_info"])
+def crossing_user(scenario: Scenario, **fields: JsonValue) -> str:
+ return scenario.user(soft_budget=SOFT_BUDGET, spend=SPEND, **fields)
+
+
+def alerts_once_a_later_key_alert_lands(
+ gateway: Gateway, alerts: AlertLog, scenario: Scenario, model: str, watched: tuple[str, ...]
+) -> tuple[str, ...]:
+ """Alerts naming a watched id, read after a key soft budget alert sent later has reached the sink."""
+ alias: Final = f"sentinel-{uuid.uuid4().hex}"
+ gateway.chat(model, key=scenario.key(key_alias=alias, soft_budget=SOFT_BUDGET, spend=SPEND))
+ (sentinel, *_) = alerts_naming(alerts, alias)
+ assert "Soft Budget Crossed" in slack_text(sentinel), sentinel
+ return tuple(body for body in alerts.bodies() if any(marker in body for marker in watched))
+
+
+def base_url(gateway: Gateway) -> str:
+ return str(gateway.client.base_url).rstrip("/")
+
+
+def upstream_requests_carrying(gateway: Gateway, marker: str) -> tuple[str, ...]:
+ observed: Final = httpx.get(f"{gateway.upstream_url}/__observations", timeout=15, trust_env=False)
+ assert observed.status_code == 200, observed.text
+ requests: Final = JSON_OBJECT.validate_json(observed.content)["requests"]
+ assert isinstance(requests, list)
+ return tuple(
+ string_value(object_value(request)["path"])
+ for request in requests
+ if marker in json.dumps(object_value(request)["body"])
+ )
+
+
+def responses_model(scenario: Scenario, gateway: Gateway, response: JsonResponse | SseResponse) -> str:
+ handle: Final = register_scenario(
+ f"user-soft-budget-{uuid.uuid4().hex}", response, control_url=gateway.upstream_url
+ )
+ scenario.cleanups.callback(delete_scenario, handle)
+ return scenario.model(api_base=f"{handle.api_base()}/v1")
+
+
+def json_responses_model(scenario: Scenario, gateway: Gateway) -> str:
+ return responses_model(scenario, gateway, JsonResponse(content_type="application/json", body=RESPONSE))
+
+
+def sse_responses_model(scenario: Scenario, gateway: Gateway) -> str:
+ frames: Final = tuple(f"event: {event['type']}\ndata: {json.dumps(event)}" for event in RESPONSE_EVENTS)
+ return responses_model(scenario, gateway, SseResponse(content_type="text/event-stream", frames=frames))
+
+
+def chat_model(scenario: Scenario, gateway: Gateway) -> str:
+ return scenario.model()
+
+
+def raw_post(url: str, key: str, path: str, body: Mapping[str, JsonValue]) -> str:
+ response: Final = httpx.post(
+ f"{url}{path}", json=dict(body), headers={"Authorization": f"Bearer {key}"}, timeout=60, trust_env=False
+ )
+ assert response.status_code == 200, response.text
+ return response.text
+
+
+def raw_stream(url: str, key: str, path: str, body: Mapping[str, JsonValue]) -> str:
+ with httpx.stream(
+ "POST",
+ f"{url}{path}",
+ json={**body, "stream": True},
+ headers={"Authorization": f"Bearer {key}"},
+ timeout=60,
+ trust_env=False,
+ ) as response:
+ text: Final = "".join(response.iter_text())
+ assert response.status_code == 200, text
+ return text
+
+
+def chat_body(model: str, marker: str) -> dict[str, JsonValue]:
+ return {"model": model, "messages": [{"role": "user", "content": marker}]}
+
+
+def messages_body(model: str, marker: str) -> dict[str, JsonValue]:
+ return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]}
+
+
+def responses_body(model: str, marker: str) -> dict[str, JsonValue]:
+ return {"model": model, "input": marker}
+
+
+def openai_client(url: str, key: str) -> OpenAI:
+ return OpenAI(
+ api_key=key, base_url=f"{url}/v1", max_retries=0, http_client=httpx.Client(timeout=60, trust_env=False)
+ )
+
+
+def chat_raw(url: str, key: str, model: str, marker: str) -> str:
+ return raw_post(url, key, "/v1/chat/completions", chat_body(model, marker))
+
+
+def chat_raw_stream(url: str, key: str, model: str, marker: str) -> str:
+ return raw_stream(url, key, "/v1/chat/completions", chat_body(model, marker))
+
+
+def chat_openai_sdk(url: str, key: str, model: str, marker: str) -> str:
+ completion: Final = openai_client(url, key).chat.completions.create(
+ model=model, messages=[{"role": "user", "content": marker}]
+ )
+ return completion.model_dump_json()
+
+
+def chat_openai_sdk_async_stream(url: str, key: str, model: str, marker: str) -> str:
+ async def consume() -> str:
+ client: Final = AsyncOpenAI(
+ api_key=key,
+ base_url=f"{url}/v1",
+ max_retries=0,
+ http_client=httpx.AsyncClient(timeout=60, trust_env=False),
+ )
+ stream: Final = await client.chat.completions.create(
+ model=model, messages=[{"role": "user", "content": marker}], stream=True
+ )
+ return "".join([chunk.model_dump_json() async for chunk in stream])
+
+ return asyncio.run(consume())
+
+
+def messages_raw(url: str, key: str, model: str, marker: str) -> str:
+ return raw_post(url, key, "/v1/messages", messages_body(model, marker))
+
+
+def messages_raw_stream(url: str, key: str, model: str, marker: str) -> str:
+ return raw_stream(url, key, "/v1/messages", messages_body(model, marker))
+
+
+def messages_anthropic_sdk(url: str, key: str, model: str, marker: str) -> str:
+ client: Final = Anthropic(
+ api_key=key, base_url=url, max_retries=0, http_client=httpx.Client(timeout=60, trust_env=False)
+ )
+ message: Final = client.messages.create(model=model, max_tokens=16, messages=[{"role": "user", "content": marker}])
+ return message.model_dump_json()
+
+
+def messages_anthropic_sdk_async_stream(url: str, key: str, model: str, marker: str) -> str:
+ async def consume() -> str:
+ client: Final = AsyncAnthropic(
+ api_key=key, base_url=url, max_retries=0, http_client=httpx.AsyncClient(timeout=60, trust_env=False)
+ )
+ async with client.messages.stream(
+ model=model, max_tokens=16, messages=[{"role": "user", "content": marker}]
+ ) as stream:
+ return "".join([event.model_dump_json() async for event in stream])
+
+ return asyncio.run(consume())
+
+
+def responses_raw(url: str, key: str, model: str, marker: str) -> str:
+ return raw_post(url, key, "/v1/responses", responses_body(model, marker))
+
+
+def responses_raw_stream(url: str, key: str, model: str, marker: str) -> str:
+ return raw_stream(url, key, "/v1/responses", responses_body(model, marker))
+
+
+def responses_openai_sdk(url: str, key: str, model: str, marker: str) -> str:
+ return openai_client(url, key).responses.create(model=model, input=marker).model_dump_json()
+
+
+def responses_openai_sdk_async_stream(url: str, key: str, model: str, marker: str) -> str:
+ async def consume() -> str:
+ client: Final = AsyncOpenAI(
+ api_key=key,
+ base_url=f"{url}/v1",
+ max_retries=0,
+ http_client=httpx.AsyncClient(timeout=60, trust_env=False),
+ )
+ stream: Final = await client.responses.create(model=model, input=marker, stream=True)
+ return "".join([event.model_dump_json() async for event in stream])
+
+ return asyncio.run(consume())
+
+
+@dataclass(frozen=True, slots=True)
+class Call:
+ model: Callable[[Scenario, Gateway], str]
+ send: Callable[[str, str, str, str], str]
+ reply: str
+
+
+CALLS: Final[Mapping[str, Call]] = {
+ "chat-httpx": Call(chat_model, chat_raw, "chat.completion"),
+ "chat-httpx-stream": Call(chat_model, chat_raw_stream, "chat.completion.chunk"),
+ "chat-openai-sdk": Call(chat_model, chat_openai_sdk, "chat.completion"),
+ "chat-openai-sdk-async-stream": Call(chat_model, chat_openai_sdk_async_stream, "chat.completion.chunk"),
+ "messages-httpx": Call(json_responses_model, messages_raw, REPLY),
+ "messages-httpx-stream": Call(sse_responses_model, messages_raw_stream, "message_stop"),
+ "messages-anthropic-sdk": Call(json_responses_model, messages_anthropic_sdk, REPLY),
+ "messages-anthropic-sdk-async-stream": Call(
+ sse_responses_model, messages_anthropic_sdk_async_stream, "message_stop"
+ ),
+ "responses-httpx": Call(json_responses_model, responses_raw, REPLY),
+ "responses-httpx-stream": Call(sse_responses_model, responses_raw_stream, "response.completed"),
+ "responses-openai-sdk": Call(json_responses_model, responses_openai_sdk, REPLY),
+ "responses-openai-sdk-async-stream": Call(
+ sse_responses_model, responses_openai_sdk_async_stream, "response.completed"
+ ),
+}
+
+
+def soft_budget_alerts_naming(alerts: AlertLog, marker: str) -> tuple[str, ...]:
+ return tuple(body for body in alerts.naming(marker) if "Message: Soft Budget Crossed" in slack_text(body))
+
+
+def exactly_one_alert_naming(alerts: AlertLog, marker: str) -> str:
+ (alert,) = held(
+ lambda: alerts.naming(marker), lambda found: len(found) == 1, holding=QUIET_SECONDS, seconds=ALERT_SECONDS
+ )
+ return alert
+
+
+def assert_user_soft_budget_alert(alert: str, user_id: str, soft_budget: float, spend: float | None = None) -> None:
+ text: Final = slack_text(alert)
+ assert f"Message: Soft Budget Crossed: Total Soft Budget:`{soft_budget}`" in text, text
+ assert f"*user_id:* `{user_id}`" in text, text
+ assert f"*soft_budget:* `{soft_budget}`" in text, text
+ assert spend is None or f"*spend:* `{spend}`" in text, text
+ assert "*event_group:* `user`" in text, text
+
+
def test_user_soft_budget_round_trips_and_stays_below_max_budget(alerting: AlertingProxy) -> None:
gateway: Final = alerting.gateway
with gateway.scenario() as scenario:
@@ -88,6 +467,128 @@ def test_user_soft_budget_round_trips_and_stays_below_max_budget(alerting: Alert
assert user_info(gateway, user_id)["soft_budget"] is None
+def test_bulk_created_user_keeps_a_valid_soft_budget_and_a_row_above_its_max_budget_is_refused(
+ alerting: AlertingProxy,
+) -> None:
+ gateway: Final = alerting.gateway
+ kept, refused = (f"integration-{uuid.uuid4().hex}" for _ in range(2))
+ with gateway.scenario() as scenario:
+ scenario.cleanups.callback(gateway.request, "POST", "/user/delete", {"user_ids": [kept, refused]})
+
+ created: Final = gateway.post(
+ "/management/v1/users/bulk",
+ {
+ "users": [
+ {"user_id": kept, "max_budget": 100, "soft_budget": 20},
+ {"user_id": refused, "max_budget": 100, "soft_budget": 150},
+ ]
+ },
+ )
+
+ assert created["meta"] == {"total_requested": 2, "created": 1, "failed": 1}, created
+ assert user_info(gateway, kept)["soft_budget"] == 20
+ assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (refused,)) == []
+
+
+def test_user_info_v2_reports_the_soft_budget(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ user_id: Final = scenario.user(max_budget=100, soft_budget=50)
+
+ info: Final = gateway.get("/v2/user/info", {"user_id": user_id})
+
+ assert (info["soft_budget"], info["max_budget"]) == (50, 100), info
+
+
+@pytest.mark.parametrize(
+ "budgets",
+ [
+ {"soft_budget": 100, "max_budget": 100},
+ {"soft_budget": 150, "max_budget": 100},
+ {"soft_budget": -1},
+ {"soft_budget": "inf"},
+ {"soft_budget": "nan"},
+ ],
+ ids=["equal-to-max", "above-max", "negative", "infinite", "not-a-number"],
+)
+def test_user_new_rejects_a_soft_budget_that_could_never_warn_before_the_max(
+ alerting: AlertingProxy, budgets: dict[str, JsonValue]
+) -> None:
+ gateway: Final = alerting.gateway
+ user_id: Final = f"integration-{uuid.uuid4().hex}"
+ try:
+ rejected: Final = gateway.request(
+ "POST", "/user/new", {"user_id": user_id, "auto_create_key": False, **budgets}
+ )
+ assert rejected.status_code == 400, rejected.text
+ assert "soft_budget" in rejected.text, rejected.text
+ assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) == []
+ finally:
+ gateway.request("POST", "/user/delete", {"user_ids": [user_id]})
+
+
+@pytest.mark.parametrize(
+ "soft_budget", ["not-a-number", [1], "", "9" * 5000 + "x"], ids=["word", "list", "empty-string", "5kb-string"]
+)
+def test_user_new_rejects_a_malformed_soft_budget_and_keeps_serving(
+ alerting: AlertingProxy, soft_budget: JsonValue
+) -> None:
+ gateway: Final = alerting.gateway
+ user_id: Final = f"integration-{uuid.uuid4().hex}"
+ with gateway.scenario() as scenario:
+ bystander: Final = scenario.user(soft_budget=50)
+ try:
+ rejected: Final = gateway.request(
+ "POST", "/user/new", {"user_id": user_id, "auto_create_key": False, "soft_budget": soft_budget}
+ )
+ assert 400 <= rejected.status_code < 500, rejected.text
+ assert "soft_budget" in rejected.text, rejected.text
+ assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) == []
+ finally:
+ gateway.request("POST", "/user/delete", {"user_ids": [user_id]})
+ assert user_info(gateway, bystander)["soft_budget"] == 50
+
+
+def test_user_update_rejects_a_max_budget_lowered_to_the_stored_soft_budget(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ user_id: Final = scenario.user(max_budget=100, soft_budget=50)
+
+ lowered: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "max_budget": 50})
+ assert lowered.status_code == 400, lowered.text
+ crossed: Final = gateway.request(
+ "POST", "/user/update", {"user_id": user_id, "max_budget": 40, "soft_budget": 45}
+ )
+ assert crossed.status_code == 400, crossed.text
+ info: Final = user_info(gateway, user_id)
+ assert (info["soft_budget"], info["max_budget"]) == (50, 100), info
+
+ gateway.post("/user/update", {"user_id": user_id, "max_budget": 40, "soft_budget": 30})
+ moved: Final = user_info(gateway, user_id)
+ assert (moved["soft_budget"], moved["max_budget"]) == (30, 40), moved
+
+
+def test_user_bulk_update_stores_a_valid_soft_budget_and_rejects_one_above_a_max_budget(
+ alerting: AlertingProxy,
+) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ roomy: Final = scenario.user(max_budget=100)
+ tight: Final = scenario.user(max_budget=10)
+
+ gateway.post("/user/bulk_update", {"users": [{"user_id": roomy, "soft_budget": 20}]})
+ assert user_info(gateway, roomy)["soft_budget"] == 20
+
+ gateway.request("POST", "/user/bulk_update", {"users": [{"user_id": tight, "soft_budget": 10}]})
+ assert user_info(gateway, tight)["soft_budget"] is None
+
+ everyone: Final = gateway.request(
+ "POST", "/user/bulk_update", {"all_users": True, "user_updates": {"soft_budget": 10}}
+ )
+ assert everyone.status_code == 400, everyone.text
+ assert (user_info(gateway, roomy)["soft_budget"], user_info(gateway, tight)["soft_budget"]) == (20, None)
+
+
@pytest.mark.parametrize("max_budget", [None, 100.0])
def test_user_over_soft_budget_gets_alert_and_request_still_succeeds(
alerting: AlertingProxy, max_budget: float | None
@@ -95,32 +596,557 @@ def test_user_over_soft_budget_gets_alert_and_request_still_succeeds(
gateway: Final = alerting.gateway
with gateway.scenario() as scenario:
model: Final = scenario.model()
- user_id: Final = scenario.user(soft_budget=0.5, max_budget=max_budget, spend=1.0)
+ user_id: Final = crossing_user(scenario, max_budget=max_budget)
key: Final = scenario.key(user_id=user_id)
response: Final = gateway.chat(model, key=key)
assert response["choices"], response
- (alert, *_) = alerts_naming(alerting.alerts, user_id, seconds=ALERT_SECONDS)
- assert "Soft Budget Crossed" in alert, alert
- assert "*soft_budget:* `0.5`" in alert, alert
+ assert_user_soft_budget_alert(exactly_one_alert_naming(alerting.alerts, user_id), user_id, SOFT_BUDGET, SPEND)
+
+
+@pytest.mark.parametrize("call", CALLS.values(), ids=CALLS.keys())
+def test_user_soft_budget_alert_fires_on_every_llm_endpoint_and_client(alerting: AlertingProxy, call: Call) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = call.model(scenario, gateway)
+ user_id: Final = crossing_user(scenario)
+ key: Final = scenario.key(user_id=user_id)
+ marker: Final = f"soft-budget-{uuid.uuid4().hex}"
+
+ received: Final = call.send(base_url(gateway), key, model, marker)
+
+ assert call.reply in received, received
+ assert len(upstream_requests_carrying(gateway, marker)) == 1
+ assert_user_soft_budget_alert(exactly_one_alert_naming(alerting.alerts, user_id), user_id, SOFT_BUDGET, SPEND)
+ logged: Final = eventually(
+ lambda: gateway.rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE "user" = %s', (user_id,)),
+ lambda rows: len(rows) > 0,
+ seconds=70,
+ )
+ assert len(logged) == 1, logged
+
+
+@pytest.mark.parametrize(("soft_budget", "spend"), [(1.0, 1.0), (0.0, 0.0)], ids=["spend-equals-soft", "zero-soft"])
+def test_user_soft_budget_alert_fires_when_spend_reaches_it_exactly(
+ alerting: AlertingProxy, soft_budget: float, spend: float
+) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = scenario.user(soft_budget=soft_budget, spend=spend)
+
+ gateway.chat(model, key=scenario.key(user_id=user_id))
+
+ assert_user_soft_budget_alert(exactly_one_alert_naming(alerting.alerts, user_id), user_id, soft_budget, spend)
def test_user_under_soft_budget_gets_no_alert(alerting: AlertingProxy) -> None:
gateway: Final = alerting.gateway
with gateway.scenario() as scenario:
model: Final = scenario.model()
- under_id: Final = scenario.user(soft_budget=50.0, spend=1.0)
+ under_id: Final = scenario.user(soft_budget=50.0, spend=SPEND)
+
gateway.chat(model, key=scenario.key(user_id=under_id))
- crossing_id: Final = scenario.user(soft_budget=0.5, spend=1.0)
- gateway.chat(model, key=scenario.key(user_id=crossing_id))
- def naming_either() -> tuple[str, ...]:
- texts: Final = (
- string_value(JSON_OBJECT.validate_json(request.body)["text"]) for request in alerting.alerts.drain()
+ assert alerts_once_a_later_key_alert_lands(gateway, alerting.alerts, scenario, model, (under_id,)) == ()
+
+
+@pytest.mark.parametrize("budgets", [{}, {"max_budget": 100.0}], ids=["no-budget", "max-budget-only"])
+def test_user_without_a_soft_budget_gets_no_alert(alerting: AlertingProxy, budgets: dict[str, JsonValue]) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = scenario.user(spend=SPEND, **budgets)
+
+ response: Final = gateway.chat(model, key=scenario.key(user_id=user_id))
+
+ assert response["choices"], response
+ assert alerts_once_a_later_key_alert_lands(gateway, alerting.alerts, scenario, model, (user_id,)) == ()
+
+
+@pytest.mark.parametrize("soft_budget", [None, SOFT_BUDGET], ids=["max-budget-only", "soft-and-max-budget"])
+def test_user_over_max_budget_is_blocked_whether_or_not_a_soft_budget_is_set(
+ alerting: AlertingProxy, soft_budget: float | None
+) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = scenario.user(max_budget=0.8, soft_budget=soft_budget, spend=SPEND)
+
+ blocked: Final = gateway.request(
+ "POST", "/v1/chat/completions", chat_body(model, "over max budget"), key=scenario.key(user_id=user_id)
+ )
+
+ assert blocked.status_code == 422, blocked.text
+ assert f"ExceededBudget: User={user_id} over budget. Spend=1.0, Budget=0.8" in blocked.text, blocked.text
+ expected: Final = 1 if soft_budget is None else 2
+ alerts: Final = held(
+ lambda: alerting.alerts.naming(user_id),
+ lambda found: len(found) == expected,
+ holding=QUIET_SECONDS,
+ seconds=ALERT_SECONDS,
+ )
+ texts: Final = tuple(slack_text(alert) for alert in alerts)
+ assert len([text for text in texts if "Budget Crossed\n Total Budget:`0.8`" in text]) == 1, texts
+ assert len([text for text in texts if "Total Soft Budget:`0.5`" in text]) == expected - 1, texts
+
+
+def test_team_key_does_not_alert_on_its_owners_soft_budget_by_default(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario)
+ key: Final = scenario.key(team_id=scenario.team(), user_id=user_id)
+
+ response: Final = gateway.chat(model, key=key)
+
+ assert response["choices"], response
+ assert alerts_once_a_later_key_alert_lands(gateway, alerting.alerts, scenario, model, (user_id,)) == ()
+
+
+def test_team_key_alerts_on_its_owners_soft_budget_when_user_budgets_apply_to_team_keys(
+ team_key_alerting: AlertingProxy,
+) -> None:
+ gateway: Final = team_key_alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario)
+ key: Final = scenario.key(team_id=scenario.team(), user_id=user_id)
+
+ response: Final = gateway.chat(model, key=key)
+
+ assert response["choices"], response
+ alert: Final = exactly_one_alert_naming(team_key_alerting.alerts, user_id)
+ assert_user_soft_budget_alert(alert, user_id, SOFT_BUDGET, SPEND)
+
+
+def test_fail_closed_budget_enforcement_still_serves_a_user_over_only_a_soft_budget(
+ fail_closed_alerting: AlertingProxy,
+) -> None:
+ gateway: Final = fail_closed_alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario)
+
+ response: Final = gateway.chat(model, key=scenario.key(user_id=user_id))
+
+ assert response["choices"], response
+ alert: Final = exactly_one_alert_naming(fail_closed_alerting.alerts, user_id)
+ assert_user_soft_budget_alert(alert, user_id, SOFT_BUDGET, SPEND)
+
+
+def test_key_soft_budget_alert_is_unchanged_for_an_owner_without_a_soft_budget(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ alias: Final = f"soft-key-{uuid.uuid4().hex}"
+ user_id: Final = scenario.user(spend=SPEND)
+ key: Final = scenario.key(user_id=user_id, key_alias=alias, soft_budget=SOFT_BUDGET, spend=SPEND)
+
+ gateway.chat(model, key=key)
+ gateway.chat(model, key=key)
+
+ (alert,) = held(
+ lambda: soft_budget_alerts_naming(alerting.alerts, user_id),
+ lambda found: len(found) == 1,
+ holding=QUIET_SECONDS,
+ seconds=ALERT_SECONDS,
+ )
+ text: Final = slack_text(alert)
+ assert "Total Soft Budget:`0.5`" in text and f"*key_alias:* `{alias}`" in text, text
+ assert "*event_group:* `key`" in text, text
+
+
+def test_team_soft_budget_alert_is_unchanged(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ call_cost: Final = 20 * 0.001 + 20 * 0.002
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
+ team_id: Final = scenario.team(
+ soft_budget=0.05, metadata={"soft_budget_alerting_emails": ["team-owner@integration.test"]}
+ )
+ key: Final = scenario.key(team_id=team_id)
+ gateway.chat(model, key=key)
+ eventually(
+ lambda: gateway.rows('SELECT spend FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)),
+ lambda rows: float(str(rows[0]["spend"])) == pytest.approx(call_cost),
+ seconds=70,
+ )
+
+ def alerts_after_another_call() -> tuple[str, ...]:
+ gateway.chat(model, key=key)
+ return soft_budget_alerts_naming(alerting.alerts, team_id)
+
+ eventually(alerts_after_another_call, lambda found: len(found) > 0, seconds=ALERT_SECONDS)
+ (alert,) = held(
+ lambda: soft_budget_alerts_naming(alerting.alerts, team_id),
+ lambda found: len(found) == 1,
+ holding=QUIET_SECONDS,
+ seconds=ALERT_SECONDS,
+ )
+ text: Final = slack_text(alert)
+ assert "Total Soft Budget:`0.05`" in text and "*event_group:* `team`" in text, text
+
+
+def test_key_and_its_owner_each_get_their_own_soft_budget_alert(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ alias: Final = f"soft-key-{uuid.uuid4().hex}"
+ user_id: Final = crossing_user(scenario)
+ key: Final = scenario.key(user_id=user_id, key_alias=alias, soft_budget=SOFT_BUDGET, spend=SPEND)
+
+ gateway.chat(model, key=key)
+ gateway.chat(model, key=key)
+
+ alerts: Final = held(
+ lambda: soft_budget_alerts_naming(alerting.alerts, user_id),
+ lambda found: len(found) == 2,
+ holding=QUIET_SECONDS,
+ seconds=ALERT_SECONDS,
+ )
+ texts: Final = tuple(slack_text(alert) for alert in alerts)
+ assert sorted("key" if "*event_group:* `key`" in text else "user" for text in texts) == ["key", "user"], texts
+
+
+def test_user_soft_budget_alerts_once_across_requests_and_keys(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario)
+ first_key: Final = scenario.key(user_id=user_id)
+ second_key: Final = scenario.key(user_id=user_id)
+
+ for key in (first_key, second_key, first_key, second_key):
+ gateway.chat(model, key=key)
+
+ assert_user_soft_budget_alert(exactly_one_alert_naming(alerting.alerts, user_id), user_id, SOFT_BUDGET, SPEND)
+
+
+def test_user_soft_budget_alert_fires_once_real_spend_crosses_it(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ call_cost: Final = 20 * 0.001 + 20 * 0.002
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
+ user_id: Final = scenario.user(soft_budget=0.05)
+ key: Final = scenario.key(user_id=user_id)
+
+ first: Final = gateway.request("POST", "/v1/chat/completions", chat_body(model, "first call"), key=key)
+ assert first.status_code == 200, first.text
+ assert float(first.headers["x-litellm-response-cost"]) == pytest.approx(call_cost)
+ eventually(
+ lambda: gateway.rows('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)),
+ lambda rows: float(str(rows[0]["spend"])) == pytest.approx(call_cost),
+ seconds=70,
+ )
+ assert alerts_once_a_later_key_alert_lands(gateway, alerting.alerts, scenario, model, (user_id,)) == ()
+
+ def alerts_after_another_call() -> tuple[str, ...]:
+ gateway.chat(model, key=key)
+ return alerting.alerts.naming(user_id)
+
+ eventually(alerts_after_another_call, lambda found: len(found) > 0, seconds=ALERT_SECONDS)
+ text: Final = slack_text(exactly_one_alert_naming(alerting.alerts, user_id))
+ assert "Total Soft Budget:`0.05`" in text and "*event_group:* `user`" in text, text
+
+
+def test_lowering_the_soft_budget_below_spend_starts_alerting(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = scenario.user(soft_budget=50.0, spend=SPEND)
+ key: Final = scenario.key(user_id=user_id)
+ gateway.chat(model, key=key)
+ assert alerts_once_a_later_key_alert_lands(gateway, alerting.alerts, scenario, model, (user_id,)) == ()
+
+ gateway.post("/user/update", {"user_id": user_id, "soft_budget": SOFT_BUDGET})
+
+ def alerts_after_another_call() -> tuple[str, ...]:
+ gateway.chat(model, key=key)
+ return alerting.alerts.naming(user_id)
+
+ eventually(alerts_after_another_call, lambda found: len(found) > 0, seconds=ALERT_SECONDS)
+ assert_user_soft_budget_alert(exactly_one_alert_naming(alerting.alerts, user_id), user_id, SOFT_BUDGET)
+
+
+def test_rejected_key_of_a_user_over_soft_budget_sends_no_alert(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario)
+ revoked: Final = gateway.post("/key/generate", {"user_id": user_id})["key"]
+ gateway.post("/key/delete", {"keys": [revoked]})
+
+ rejected: Final = gateway.request(
+ "POST", "/v1/chat/completions", chat_body(model, "revoked key"), key=string_value(revoked)
+ )
+
+ assert rejected.status_code == 401, rejected.text
+ assert alerts_once_a_later_key_alert_lands(gateway, alerting.alerts, scenario, model, (user_id,)) == ()
+
+
+def test_alert_sink_refusing_the_alert_never_fails_the_request(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario)
+ alerting.sink.refusing.set()
+ try:
+ response: Final = gateway.chat(model, key=scenario.key(user_id=user_id))
+ assert response["choices"], response
+ assert_user_soft_budget_alert(
+ exactly_one_alert_naming(alerting.alerts, user_id), user_id, SOFT_BUDGET, SPEND
)
- return tuple(text for text in texts if under_id in text or crossing_id in text)
+ assert gateway.chat(model, key=scenario.key())["choices"]
+ finally:
+ alerting.sink.refusing.clear()
+ other_id: Final = crossing_user(scenario)
+ gateway.chat(model, key=scenario.key(user_id=other_id))
+ assert_user_soft_budget_alert(exactly_one_alert_naming(alerting.alerts, other_id), other_id, SOFT_BUDGET, SPEND)
- seen: Final = eventually(naming_either, lambda found: len(found) > 0, seconds=ALERT_SECONDS)
- assert all(under_id not in text for text in seen), seen
- assert any(crossing_id in text for text in seen), seen
+
+def test_alert_queued_while_the_sink_stalls_is_delivered_once_the_sink_answers(alerting: AlertingProxy) -> None:
+ pytest.skip("BUG: a Slack alert queued while an earlier alert is still being posted is dropped and never sent")
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ stalled, queued = (f"soft-key-{uuid.uuid4().hex}" for _ in range(2))
+ alerting.sink.stalling.set()
+ try:
+ gateway.chat(model, key=scenario.key(key_alias=stalled, soft_budget=SOFT_BUDGET, spend=SPEND))
+ alerts_naming(alerting.alerts, stalled)
+ gateway.chat(model, key=scenario.key(key_alias=queued, soft_budget=SOFT_BUDGET, spend=SPEND))
+ finally:
+ alerting.sink.stalling.clear()
+ alerting.sink.release.set()
+ try:
+ assert len(alerts_naming(alerting.alerts, queued)) == 1
+ finally:
+ alerting.sink.release.clear()
+
+
+def test_stalled_alert_sink_does_not_hold_the_request_or_duplicate_the_alert(alerting: AlertingProxy) -> None:
+ gateway: Final = alerting.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ stalled_id: Final = crossing_user(scenario)
+ alerting.sink.stalling.set()
+ try:
+ assert gateway.chat(model, key=scenario.key(user_id=stalled_id))["choices"]
+ alerts_naming(alerting.alerts, stalled_id)
+ assert gateway.chat(model, key=scenario.key())["choices"]
+ assert not alerting.sink.release.is_set()
+ finally:
+ alerting.sink.stalling.clear()
+ alerting.sink.release.set()
+ try:
+ later_id: Final = crossing_user(scenario)
+ gateway.chat(model, key=scenario.key(user_id=later_id))
+ for user_id in (later_id, stalled_id):
+ assert_user_soft_budget_alert(
+ exactly_one_alert_naming(alerting.alerts, user_id), user_id, SOFT_BUDGET, SPEND
+ )
+ finally:
+ alerting.sink.release.clear()
+
+
+def mail_to(mailbox: Mailbox, recipient: str) -> tuple[str, ...]:
+ return tuple(delivery.html for delivery in mailbox.deliveries() if recipient in delivery.recipients)
+
+
+def test_user_soft_budget_webhook_event_and_email_name_the_user_and_budget(mailing: MailingProxy) -> None:
+ gateway: Final = mailing.gateway
+ email: Final = f"owner-{uuid.uuid4().hex}@integration.test"
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario, user_email=email)
+
+ key: Final = scenario.key(user_id=user_id)
+
+ response: Final = gateway.chat(model, key=key)
+ gateway.chat(model, key=key)
+
+ assert response["choices"], response
+ event: Final = JSON_OBJECT.validate_json(exactly_one_alert_naming(mailing.webhooks, user_id))
+ assert {
+ name: event[name]
+ for name in ("event", "event_group", "user_id", "user_email", "soft_budget", "spend", "max_budget", "token")
+ } == {
+ "event": "soft_budget_crossed",
+ "event_group": "user",
+ "user_id": user_id,
+ "user_email": email,
+ "soft_budget": SOFT_BUDGET,
+ "spend": SPEND,
+ "max_budget": None,
+ "token": None,
+ }, event
+ mails: Final = held(
+ lambda: mail_to(mailing.mailbox, email),
+ lambda found: len(found) == 1,
+ holding=QUIET_SECONDS,
+ seconds=ALERT_SECONDS,
+ )
+ assert len(mails) == 1, mails
+ assert "Your LiteLLM account has crossed its soft budget limit of $0.5" in mails[0], mails
+ assert "Current Spend: $1.0" in mails[0], mails
+
+
+def test_key_soft_budget_still_sends_the_generic_and_the_api_key_email(mailing: MailingProxy) -> None:
+ gateway: Final = mailing.gateway
+ email: Final = f"owner-{uuid.uuid4().hex}@integration.test"
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = scenario.user(user_email=email, spend=SPEND)
+ key: Final = scenario.key(user_id=user_id, soft_budget=SOFT_BUDGET, spend=SPEND)
+
+ gateway.chat(model, key=key)
+
+ mails: Final = held(
+ lambda: mail_to(mailing.mailbox, email),
+ lambda found: len(found) == 2,
+ holding=QUIET_SECONDS,
+ seconds=ALERT_SECONDS,
+ )
+ assert sorted("generic" if mail == "Alert from LiteLLM Server" else "soft budget" for mail in mails) == [
+ "generic",
+ "soft budget",
+ ], mails
+ assert any("Your LiteLLM API key has crossed its soft budget limit of $0.5" in mail for mail in mails)
+
+
+def test_key_soft_budget_of_an_owner_without_an_email_reaches_the_webhook_once(mailing: MailingProxy) -> None:
+ pytest.skip("BUG: a key soft budget crossing whose owner has no email re-sends the webhook event on every request")
+ gateway: Final = mailing.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ alias: Final = f"soft-key-{uuid.uuid4().hex}"
+ key: Final = scenario.key(
+ user_id=scenario.user(spend=SPEND), key_alias=alias, soft_budget=SOFT_BUDGET, spend=SPEND
+ )
+
+ for _ in range(3):
+ assert gateway.chat(model, key=key)["choices"]
+
+ events: Final = held(
+ lambda: tuple(body for body in mailing.webhooks.naming(alias) if '"soft_budget_crossed"' in body),
+ lambda found: len(found) == 1,
+ holding=QUIET_SECONDS,
+ seconds=ALERT_SECONDS,
+ )
+ assert JSON_OBJECT.validate_json(events[0])["event_group"] == "key", events
+
+
+def test_user_without_an_email_is_served_and_still_reaches_the_webhook(mailing: MailingProxy) -> None:
+ gateway: Final = mailing.gateway
+ with gateway.scenario() as scenario:
+ model: Final = scenario.model()
+ user_id: Final = crossing_user(scenario)
+ key: Final = scenario.key(user_id=user_id)
+
+ for _ in range(3):
+ assert gateway.chat(model, key=key)["choices"]
+
+ event: Final = JSON_OBJECT.validate_json(exactly_one_alert_naming(mailing.webhooks, user_id))
+ assert (event["event"], event["event_group"], event["user_email"]) == ("soft_budget_crossed", "user", None)
+ assert [html for delivery in mailing.mailbox.deliveries() if user_id in (html := delivery.html)] == []
+
+
+@dataclass(frozen=True, slots=True)
+class Job:
+ send: Callable[[str, str, str, str], str]
+ model: str
+ key: str
+ user_id: str
+
+
+def burst_jobs(scenario: Scenario, gateway: Gateway, users: int, requests: int) -> tuple[Job, ...]:
+ chat: Final = scenario.model()
+ json_model: Final = json_responses_model(scenario, gateway)
+ sse_model: Final = sse_responses_model(scenario, gateway)
+ routes: Final = (
+ (chat_raw, chat),
+ (chat_raw_stream, chat),
+ (messages_raw, json_model),
+ (messages_raw_stream, sse_model),
+ (responses_raw, json_model),
+ (responses_raw_stream, sse_model),
+ )
+ user_ids: Final = tuple(crossing_user(scenario) for _ in range(users))
+ keys: Final = tuple(scenario.key(user_id=user_id) for user_id in user_ids)
+ return tuple(
+ Job(
+ routes[index % len(routes)][0], routes[index % len(routes)][1], keys[index % users], user_ids[index % users]
+ )
+ for index in range(requests)
+ )
+
+
+def alerts_per_user(bodies: tuple[str, ...], user_ids: frozenset[str]) -> Mapping[str, int]:
+ return {user_id: len([body for body in bodies if user_id in body]) for user_id in user_ids}
+
+
+def test_alert_sink_outage_mid_burst_fails_no_request_and_alerting_resumes(
+ chaos: ChaosProxy, record_property: Callable[[str, object], None]
+) -> None:
+ gateway: Final = chaos.proxy.gateway
+ url: Final = base_url(gateway)
+ with gateway.scenario() as scenario:
+ jobs: Final = burst_jobs(scenario, gateway, users=10, requests=30)
+ user_ids: Final = frozenset(job.user_id for job in jobs)
+ with ThreadPoolExecutor(max_workers=10) as pool:
+ with wire_server(accept_alert, port=chaos.sink_port) as before:
+ sent: Final = tuple(
+ pool.submit(job.send, url, job.key, job.model, f"burst-{uuid.uuid4().hex}") for job in jobs
+ )
+ wait(sent, return_when=FIRST_COMPLETED)
+ before_outage: Final = AlertLog(before)
+ received: Final = tuple(future.result() for future in sent)
+ assert len(received) == 30 and all(received), received
+
+ with wire_server(accept_alert, port=chaos.sink_port) as after:
+ after_outage: Final = AlertLog(after)
+ late_id: Final = crossing_user(scenario)
+ assert gateway.chat(scenario.model(), key=scenario.key(user_id=late_id))["choices"]
+ assert_user_soft_budget_alert(exactly_one_alert_naming(after_outage, late_id), late_id, SOFT_BUDGET, SPEND)
+ delivered: Final = alerts_per_user((*before_outage.bodies(), *after_outage.bodies()), user_ids)
+ record_property("burst_users_alerted", sum(delivered.values()))
+ assert all(count <= 1 for count in delivered.values()), delivered
+
+
+def status_or_disconnect(url: str, key: str, model: str) -> int:
+ try:
+ return httpx.post(
+ f"{url}/v1/chat/completions",
+ json=chat_body(model, "worker kill burst"),
+ headers={"Authorization": f"Bearer {key}", "connection": "close"},
+ timeout=60,
+ trust_env=False,
+ ).status_code
+ except httpx.TransportError:
+ return 0
+
+
+def test_user_soft_budget_alert_still_fires_after_a_worker_is_killed_mid_burst(chaos: ChaosProxy) -> None:
+ proxy: Final = chaos.proxy
+ gateway: Final = proxy.gateway
+ url: Final = base_url(gateway)
+ with wire_server(accept_alert, port=chaos.sink_port) as sink, gateway.scenario() as scenario:
+ alerts: Final = AlertLog(sink)
+ model: Final = scenario.model()
+ user_ids: Final = tuple(crossing_user(scenario) for _ in range(6))
+ keys: Final = tuple(scenario.key(user_id=user_id) for user_id in user_ids)
+ victim: Final = min(proxy.worker_pids())
+ with ThreadPoolExecutor(max_workers=8) as pool:
+ sent: Final = tuple(pool.submit(status_or_disconnect, url, keys[index % 6], model) for index in range(24))
+ proxy.kill_worker(victim)
+ statuses: Final = tuple(future.result() for future in sent)
+ assert set(statuses) <= {200, 0} and 200 in statuses, statuses
+ assert victim not in proxy.worker_pids()
+
+ late_id: Final = crossing_user(scenario)
+ assert gateway.chat(model, key=scenario.key(user_id=late_id))["choices"]
+ assert_user_soft_budget_alert(exactly_one_alert_naming(alerts, late_id), late_id, SOFT_BUDGET, SPEND)
+ delivered: Final = alerts_per_user(alerts.bodies(), frozenset(user_ids))
+ assert all(count <= 1 for count in delivered.values()), delivered