scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 199

fix(router): keep DB-only deployments priced across price data reloads (#48486)

Browse source 
* fix(router): track every router for price data reloads so DB-only deployments keep custom pricing

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(router): register the router for reload replay only once model_list exists, and type the regression test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: joshua <joshua@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-26 17:17:52 -07:00 • committed by GitHub
parent 833d82bb4e
commit 549e6d4f46
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 34 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/router.py
View file

@ -1483,15 +1483,16 @@ class Router:
if model_list is not None:
# set_model_list will build indices automatically
self.set_model_list(model_list)
# Track this router so a price data reload can rebuild its deployments'
# cost-map entries from the list it is serving at that moment.
_live_routers.add(self)
self.healthy_deployments: list = self.model_list
for m in model_list:
if "model" in m["litellm_params"]:
self.deployment_latency_map[m["litellm_params"]["model"]] = 0
else:
self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to work
# Track this router so a price data reload can rebuild its deployments'
# cost-map entries from the list it is serving at that moment, whether
# those deployments came from model_list or were added later from the DB.
_live_routers.add(self)
if allowed_fails is not None:
self.allowed_fails = allowed_fails

30
tests/unit/test_router_model_cost_isolation.py
View file

@ -3081,6 +3081,36 @@ def test_reload_keeps_custom_pricing_configured_on_litellm_params_for_a_db_model
_invalidate_model_cost_lowercase_map()
def test_reload_keeps_custom_pricing_of_a_db_model_on_a_router_built_without_a_model_list(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
A proxy whose config.yaml has no model_list builds its router without one and
takes every deployment from the DB afterwards. A price data reload must restore
that deployment's custom pricing exactly as it does for a config router.
"""
fetched_catalog: Final = copy.deepcopy(litellm.model_cost)
router: Final = Router()
router.add_deployment(
deployment=Deployment(
model_name="db-only-priced",
litellm_params=LiteLLM_Params(
model="openai/gpt-4o",
api_key="sk-fake",
input_cost_per_token=0.000123,
output_cost_per_token=0.000456,
),
model_info=ModelInfo(id="db-only-priced-id"),
)
)
assert litellm.model_cost["db-only-priced-id"]["input_cost_per_token"] == 0.000123
_simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog)
assert litellm.model_cost["db-only-priced-id"]["input_cost_per_token"] == 0.000123
assert litellm.model_cost["db-only-priced-id"]["output_cost_per_token"] == 0.000456
def test_replay_live_router_model_cost_rebuilds_every_live_router():
"""
A process can hold more than one Router, so the rebuild has to fan out across