test(integration): read failure spend rows from the owned proxy database and keep the restart cell on shared state
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Main now gives each owned proxy a private database, so the spend row reads go through the proxy gateway.
The restart cell boots both proxies on the worker state so the second one keeps the stored model, and it
waits for the post-restart events while that proxy is still up since batch loggers do not flush on shutdown
This commit is contained in:
yucheng-berri 2026-10-24 02:50:42 -07:00
parent 74ab6f1df7
commit 89ac9b3ce4
2 changed files with 44 additions and 38 deletions

View file

@ -15,7 +15,6 @@ import openai
import pytest
import yaml
from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, Wire, wire_server
@ -227,9 +226,9 @@ def _single_failure_event(rig: Rig, model: str) -> dict[str, JsonValue]:
return events[0]
def _spend_row(call_id: str) -> dict[str, JsonValue]:
def _spend_row(proxy: Gateway, call_id: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
lambda: proxy.rows(
'SELECT status, messages, response, proxy_server_request, metadata FROM "LiteLLM_SpendLogs" '
"WHERE request_id=%s",
(call_id,),
@ -264,7 +263,7 @@ def test_provider_error_echoing_the_prompt_is_redacted_in_callbacks_and_spend_lo
assert error_information["error_code"] == "400", error_information
assert error_information["llm_provider"] == "openai", error_information
assert "hidden_params" not in event, sorted(event)
row: Final = _spend_row(response.headers["x-litellm-call-id"])
row: Final = _spend_row(rig.proxy, response.headers["x-litellm-call-id"])
assert row["status"] == "failure", row
assert secret not in json.dumps(row, default=str), row
persisted: Final = object_value(
@ -361,7 +360,7 @@ def _assert_failure_redacted(rig: Rig, model: str, secret: str, call_id: str) ->
error_information: Final = _error_information(event)
assert error_information["error_class"], error_information
assert error_information["llm_provider"], error_information
row: Final = _spend_row(call_id)
row: Final = _spend_row(rig.proxy, call_id)
assert row["status"] == "failure", row
assert secret not in json.dumps(row, default=str), row
return event
@ -371,7 +370,7 @@ def _assert_failure_raw(rig: Rig, model: str, secret: str, call_id: str | None =
event: Final = _single_failure_event(rig, model)
assert secret in json.dumps(event), json.dumps(event)
if call_id is not None:
row: Final = _spend_row(call_id)
row: Final = _spend_row(rig.proxy, call_id)
assert secret in json.dumps(row, default=str), row
return event
@ -601,7 +600,7 @@ def test_a9_unknown_model_failure_redacted(rig: Rig) -> None:
assert rig.provider.drain() == ()
event: Final = eventually(lambda: rig.failure_events(model), lambda values: len(values) >= 1, seconds=20)[0]
assert secret not in json.dumps(event), json.dumps(event)
row: Final = _spend_row(_call_id(response))
row: Final = _spend_row(rig.proxy, _call_id(response))
assert secret not in json.dumps(row, default=str), row
@ -904,7 +903,7 @@ def test_e4_header_only_unknown_model_spend_row_never_carries_marker(rig_off: Ri
headers={"x-litellm-enable-message-redaction": "true"},
)
assert response.status_code == 400, response.text
row: Final = _spend_row(_call_id(response))
row: Final = _spend_row(rig_off.proxy, _call_id(response))
assert secret not in json.dumps(row, default=str), row
@ -1107,4 +1106,4 @@ def test_f4_five_identical_failures_log_once_each(rig: Rig) -> None:
lambda values: len(values) == 1,
seconds=30,
)
assert _spend_row(call_id)["status"] == "failure", call_id
assert _spend_row(rig.proxy, call_id)["status"] == "failure", call_id

View file

@ -11,7 +11,7 @@ import httpx
import pytest
import yaml
from integration._support.client import Gateway, JsonValue, eventually, gateway_from_environment, object_value
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.process import OwnedProxy, OwnedState, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
BURST: Final = 30
@ -190,6 +190,7 @@ def test_g2_slow_sink_no_deadlock_no_duplicates(tmp_path: Path) -> None:
@pytest.mark.timeout(280)
def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None:
marker: Final = uuid.uuid4().hex
model: Final = f"restart-model-{uuid.uuid4().hex}"
with wire_server(_provider) as provider, wire_server(lambda _: Reply()) as endpoint:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["litellm_settings"].update(
@ -199,12 +200,14 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None:
path.write_text(yaml.safe_dump(config))
overrides: Final = {"GENERIC_LOGGER_ENDPOINT": endpoint.url}
with gateway_from_environment() as gateway:
bodies: Final = _bodies("restart-model", marker)
with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned_one:
bodies: Final = _bodies(model, marker)
with owned_proxy_process(
gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER
) as owned_one:
owned_one.gateway.post(
"/model/new",
{
"model_name": "restart-model",
"model_name": model,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_base": provider.url + "/v1",
@ -218,20 +221,22 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None:
owned_one,
provider,
endpoint,
"restart-model",
model,
threading.Event(),
threading.Event(),
[],
),
bodies[: BURST // 2],
)
with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned_two:
with owned_proxy_process(
gateway, tmp_path, overrides, config=path, workers=2, state=OwnedState.SHARED_WITH_WORKER
) as owned_two:
rig_two: Final = Rig(
owned_two.gateway,
owned_two,
provider,
endpoint,
"restart-model",
model,
threading.Event(),
threading.Event(),
[],
@ -249,33 +254,35 @@ def test_g3_proxy_restart_mid_burst(tmp_path: Path) -> None:
eventually(served, lambda statuses: statuses[0] == 400, seconds=60)
second: Final = _fire(rig_two, bodies[BURST // 2 :])
answered: Final = first + second + tuple(probes)
assert all(status in (400, 500) for status, _ in answered), answered
post_restart_ids: Final = frozenset(cid for status, cid in second + tuple(probes) if status == 400 and cid)
assert post_restart_ids, answered
assert any(marker.encode() in request.body for request in provider.drain())
collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones
answered: Final = first + second + tuple(probes)
assert all(status in (400, 500) for status, _ in answered), answered
post_restart_ids: Final = frozenset(
cid for status, cid in second + tuple(probes) if status == 400 and cid
)
assert post_restart_ids, answered
assert any(marker.encode() in request.body for request in provider.drain())
collected: list[Request] = [] # mutable-ok: drain consumes batches, later polls keep earlier ones
def landed() -> tuple[str, ...]:
collected.extend(endpoint.drain())
return tuple(
str(event.get("litellm_call_id"))
def landed() -> tuple[str, ...]:
collected.extend(endpoint.drain())
return tuple(
str(event.get("litellm_call_id"))
for batch in collected
if _json_body_ok(batch.body)
for event in json.loads(batch.body)
)
landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60)
assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected
assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids)
events: Final = tuple(
object_value(event)
for batch in collected
if _json_body_ok(batch.body)
for event in json.loads(batch.body)
)
landed_ids: Final = eventually(landed, lambda ids: post_restart_ids <= set(ids), seconds=60)
assert all(not batch.body.strip() for batch in collected if not _json_body_ok(batch.body)), collected
assert len(landed_ids) == len(set(landed_ids)), ("duplicate events after restart", landed_ids)
events: Final = tuple(
object_value(event)
for batch in collected
if _json_body_ok(batch.body)
for event in json.loads(batch.body)
)
for event in events:
assert marker not in json.dumps(event), json.dumps(event)[:400]
for event in events:
assert marker not in json.dumps(event), json.dumps(event)[:400]
@pytest.mark.timeout(280)