scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(e2e): return create_model on first /v1/models listing

Browse source 
Continuous 30s listing after first sight made the suite crawl: every
create_model slept through repeated GET /v1/models, and multi-worker misses
could reset the window. Keep the 40s first-listing budget; set
model_servable_db_sync_seconds to 0 so we return as soon as the model appears
This commit is contained in:
mubashir1osmani 2026-07-28 18:23:52 -07:00
parent 7ff2bcbf14
commit c76b003c6f
1 changed files with 7 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
tests/e2e/proxy_client.py
View file

@ -75,17 +75,13 @@ from transport import HttpTransport, SplitTransport, Transport
RowsPredicate = Callable[[list[SpendLogRow]], bool]
# After /model/new, the control-plane writer reloads itself immediately, but every
# other gateway worker (and peer pod) only picks the model up on its add_deployment
# job. That job runs every proxy_config_reload_interval_seconds (product default 30).
# A single /v1/models hit can land on a hot worker while the next /chat hits a cold
# one ("Invalid model name"). Wait for first listing within MODEL_SERVABLE_TIMEOUT,
# then require continuous listing for MODEL_SERVABLE_DB_SYNC_SECONDS (the default
# reload interval) so every worker has had a chance to sync from the DB.
# After /model/new, poll data-plane /v1/models until the model is listed (or fail).
# Bound by MODEL_SERVABLE_TIMEOUT so a stuck reload does not burn the spend
# poll_timeout (120s). Return on first listing: a continuous multi-worker sync
# wait (30s per create_model) made the suite unusably slow on stage.
MODEL_SERVABLE_TIMEOUT = 40.0
MODEL_SERVABLE_DB_SYNC_SECONDS = 30.0
MODEL_SERVABLE_DB_SYNC_SECONDS = 0.0
MODEL_SERVABLE_INTERVAL = 2.0
# Cap each /v1/models poll so one slow request cannot outlast the remaining budget.
MODEL_SERVABLE_REQUEST_TIMEOUT = 5.0
@ -288,9 +284,7 @@ class ProxyClient:
it is already present on the first poll, so this adds one request.
First listing must arrive within `model_servable_timeout` (not the longer
spend `poll_timeout`). The model must then stay listed for
`model_servable_db_sync_seconds` (product default DB reload interval) so every
gateway worker has run add_deployment before callers use the model."""
spend `poll_timeout`). Returns as soon as /v1/models lists the model."""
model_id = unwrap(
self.transport.post(
"/model/new",
@ -307,10 +301,7 @@ class ProxyClient:
return model_id
def _await_model_servable(self, model_name: str) -> None:
"""Block until the data plane lists `model_name` long enough for DB sync.
Fails if first listing misses model_servable_timeout, or if continuous listing
for model_servable_db_sync_seconds never holds (multi-worker / peer reload)."""
"""Block until the data plane lists `model_name`, or fail at model_servable_timeout."""
outcome = await_servable(
lambda poll_timeout: self.transport.get(
"/v1/models",