fix(router): replay each distinct cost write once and keep in-place edits through a rebuild

This commit is contained in:
mateo-berri 2026-11-02 23:31:47 -08:00
parent 8a9a8a5f71
commit 979d01db6c
4 changed files with 256 additions and 30 deletions

View file

@ -570,6 +570,7 @@ class AliasedCostEntry(dict[str, object]):
"registered_fields",
"saved_write_fields",
"written_fields",
"written_values",
)
def __init__(
@ -583,6 +584,7 @@ class AliasedCostEntry(dict[str, object]):
displaced_values: tuple[tuple[str, object], ...] = (),
saved_write_fields: frozenset[str] = frozenset(),
latest_unsaved_write_fields: frozenset[str] = frozenset(),
written_values: tuple[tuple[str, object], ...] = (),
) -> None:
super().__init__(entry)
self.names: tuple[str, ...] = names
@ -593,6 +595,7 @@ class AliasedCostEntry(dict[str, object]):
self.displaced_values: tuple[tuple[str, object], ...] = displaced_values
self.saved_write_fields: frozenset[str] = saved_write_fields
self.latest_unsaved_write_fields: frozenset[str] = latest_unsaved_write_fields
self.written_values: tuple[tuple[str, object], ...] = written_values
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:

View file

@ -517,11 +517,14 @@ from litellm.utils import (
is_generalized_model_info,
is_region_allowed,
lookup_model_info_for_registration,
model_cost_fields_set_in_place,
model_cost_keys_written_onto_other_rows,
model_cost_registration_key,
model_cost_row_writers,
model_cost_rows_written_by,
model_info_with_pricing_overrides,
provider_rejectable_params,
put_back_model_cost_fields_set_in_place,
restore_model_cost_rows_to_catalog,
runtime_model_cost_registrations_on,
runtime_model_cost_registrations_written_last,
@ -1138,10 +1141,14 @@ class _SharedCostWrite:
def lands_on(self, writers: frozenset[str]) -> bool:
return self.model_id in writers or not writers.isdisjoint(self.backend_keys)
def replay_key(self, stale: frozenset[str]) -> tuple[str | None, str]:
"""Equal for two entries only when replaying both writes the ``stale`` rows exactly as replaying one does. A
shared row never takes a price, only what a price being set pulls in from the backend model, so prices count
only as set or unset, and a stale id row keeps its write apart by its id."""
@property
def sets_mode(self) -> bool:
return isinstance(self.model_info.get("mode"), str)
def replay_key(self, targets: frozenset[str]) -> tuple[str | None, str]:
"""Equal for two entries only when replaying both writes the ``targets`` rows exactly as replaying one does.
A shared row never takes a price, only what a price being set pulls in from the backend model, so prices count
only as set or unset, and an id that names a target row or wrote onto one keeps its write apart by its id."""
write_inputs: Final = (
self.model,
self.custom_llm_provider,
@ -1149,7 +1156,7 @@ class _SharedCostWrite:
_SharedCostWrite._write_inputs(self.litellm_params, _LITELLM_PARAMS_COST_WRITE_INPUTS),
)
return (
self.model_id if self.model_id in stale else None,
self.model_id if self.model_id in targets else None,
json.dumps(write_inputs, sort_keys=True, default=repr),
)
@ -12128,7 +12135,11 @@ class PythonCore(DeprecatedRouterMembers):
# OLD model_name/tags) before the re-add below re-registers it.
self._unregister_pre_routing_strategy_for_deployment(deployment=_deployment_on_router)
self._rebuild_model_cost_rows(
lambda: self._rebuild_cost_rows_written_by(PythonCore._backend_write_keys(_deployment_on_router))
lambda: self._rebuild_cost_rows_written_by(
PythonCore._shared_write_keys(
(PythonCore._deployment_shared_cost_write(_deployment_on_router),)
)
)
)
# if the model_id is not in router
@ -12265,7 +12276,8 @@ class PythonCore(DeprecatedRouterMembers):
Takes only a ``Deployment``, so it registers pricing for a deployment that
is never added to ``self._core_model_list`` (a per-request client-side-credential
deployment) just as readily as one that is. ``onto`` limits the writes to those rows.
deployment) just as readily as one that is. ``onto`` limits the writes to those rows, and the id's write to an
id it names, which is how a rebuild replays an id whose write landed on a rebuilt row named otherwise.
"""
PythonCore._register_deployment_in_model_cost(
model_id=deployment.model_info.id,
@ -12391,13 +12403,14 @@ class PythonCore(DeprecatedRouterMembers):
)
@staticmethod
def _backend_write_keys(deployment: Deployment) -> frozenset[str]:
return frozenset(
PythonCore._backend_cost_map_keys(
model=deployment.litellm_params.model,
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
)
)
def _shared_write_keys(writes: Iterable[_SharedCostWrite]) -> frozenset[str]:
"""The ``register_model`` keys of ``writes`` whose rows other deployments read too: every backend key, and
each id whose write landed on a row named otherwise, such as the ``gpt-4o`` row an ``openai/gpt-4o`` id
resolves to."""
listed: Final = tuple(writes)
ids: Final = frozenset(write.model_id for write in listed if write.model_id is not None)
backend_keys: Final = chain.from_iterable(write.backend_keys for write in listed)
return frozenset(backend_keys) | model_cost_keys_written_onto_other_rows(ids)
def _rebuild_rows_a_replaced_model_list_wrote(self, previous: Iterable[object]) -> None:
"""Rebuild the shared rows a replaced ``model_list`` wrote and trim the id rows of the deployments still
@ -12406,7 +12419,7 @@ class PythonCore(DeprecatedRouterMembers):
writes: Final = tuple(
write for write in map(PythonCore._entry_shared_cost_write, previous) if write is not None
)
self._rebuild_cost_rows_written_by(frozenset(chain.from_iterable(write.backend_keys for write in writes)))
self._rebuild_cost_rows_written_by(PythonCore._shared_write_keys(writes))
self._give_back_id_row_fields(
write.model_id
for write in writes
@ -12458,39 +12471,55 @@ class PythonCore(DeprecatedRouterMembers):
def _rebuild_cost_rows_written_by(self, written_keys: frozenset[str]) -> None:
"""Rebuild the rows ``register_model`` writes of ``written_keys`` landed on: the loaded catalog row, then the
``register_model`` registrations replayed on every reload, then every live deployment that wrote it in
``model_list`` order, then again the fields of those registrations no deployment wrote over since, so each
field keeps the order its writes came in, whether the registration came before the router or after it. The
replay writes only those rows, so a row it leaves alone keeps what later deployments and registrations put
there, and a run of deployments with the same write replays it once, which leaves the rows as replaying
every one of them does."""
``model_list`` order, then again the fields of those registrations no deployment wrote over since, then the
fields set in place after their last write, so each field keeps the order its writes came in, whether the
registration came before the router or after it. The replay writes only those rows, so a row it leaves alone
keeps what later deployments and registrations put there, and deployments with the same write replay it once,
which leaves the rows as replaying every one of them does."""
stale: Final = model_cost_rows_written_by(written_keys) if written_keys else frozenset[str]()
if not stale:
return
writers: Final = model_cost_row_writers(stale)
targets: Final = stale | writers
routers: Final = tuple(_live_routers)
entries: Final = tuple(chain.from_iterable(router._model_list_entries() for router in routers))
writes: Final = (PythonCore._entry_shared_cost_write(entry) for entry in entries)
landing: Final = tuple(
(position, write.replay_key(stale))
for position, write in enumerate(writes)
if write is not None and write.lands_on(writers)
(position, write) for position, write in enumerate(writes) if write is not None and write.lands_on(writers)
)
replay_runs: Final = groupby(landing, key=itemgetter(1))
replayed: Final = tuple(
PythonCore._buildable_deployments(entries[tuple(run)[-1][0]] for _replay_key, run in replay_runs)
PythonCore._buildable_deployments(
entries[position] for position in PythonCore._replayed_positions(landing, targets)
)
)
persisted: Final = runtime_model_cost_registrations_on(stale)
written_last: Final = runtime_model_cost_registrations_written_last(persisted)
set_in_place: Final = model_cost_fields_set_in_place(stale)
restore_model_cost_rows_to_catalog(stale)
if persisted:
self._host.invoke("register_model")(model_cost=dict(persisted))
for deployment in replayed:
PythonCore._register_deployment_pricing(deployment, onto=stale, host=self._host)
PythonCore._register_deployment_pricing(deployment, onto=targets, host=self._host)
if written_last:
self._host.invoke("register_model")(model_cost=dict(written_last))
if set_in_place:
put_back_model_cost_fields_set_in_place(set_in_place)
for router in routers:
router._invalidate_model_group_info_cache()
@staticmethod
def _replayed_positions(
landing: tuple[tuple[int, _SharedCostWrite], ...], targets: frozenset[str]
) -> tuple[int, ...]:
"""The position of the last of each set of ``landing`` writes that leave the ``targets`` rows as one of them
does. Each field keeps the last value written to it, so only the last write of each set counts, unless a write
sets ``mode``: what a ``mode`` write does depends on the mode the row holds, so then only back-to-back runs
collapse."""
keyed: Final = tuple((position, write.replay_key(targets)) for position, write in landing)
if any(write.sets_mode for _position, write in landing):
return tuple(tuple(run)[-1][0] for _replay_key, run in groupby(keyed, key=itemgetter(1)))
return tuple(sorted({replay_key: position for position, replay_key in keyed}.values()))
def _core_delete_deployment(self, id: str) -> Deployment | None:
"""
Parameters:
@ -12526,7 +12555,9 @@ class PythonCore(DeprecatedRouterMembers):
id,
)
self._rebuild_model_cost_rows(
lambda: self._rebuild_cost_rows_written_by(PythonCore._backend_write_keys(removed_deployment()))
lambda: self._rebuild_cost_rows_written_by(
PythonCore._shared_write_keys((PythonCore._deployment_shared_cost_write(removed_deployment()),))
)
)
except Exception:
return None

View file

@ -3810,6 +3810,7 @@ def _replace_cost_map_entry(
}.items()
)
created_by_registration: Final = previous.created_by_registration if current is previous else current is None
written_now: Final = {name: value for name, value in entry.items() if name in changed_fields or name in set_fields}
cost_map.update(
dict.fromkeys(
(key, *_names_sharing_entry(cost_map, key)),
@ -3825,6 +3826,10 @@ def _replace_cost_map_entry(
previous.saved_write_fields | set_fields if saved else previous.saved_write_fields - set_fields
),
latest_unsaved_write_fields=previous.latest_unsaved_write_fields if saved else set_fields,
written_values=(
*((name, value) for name, value in previous.written_values if name not in written_now),
*written_now.items(),
),
),
)
)
@ -4110,6 +4115,17 @@ def model_cost_rows_written_by(keys: Collection[str]) -> frozenset[str]:
)
def model_cost_keys_written_onto_other_rows(keys: Collection[str]) -> frozenset[str]:
"""Each of ``keys`` whose ``register_model`` write landed on a row named otherwise, as an ``openai/gpt-4o``
write lands on the ``gpt-4o`` row."""
cost_map: Final = _builtin_cost_map()
entries: Final = (entry for entry in tuple(cost_map.values()) if isinstance(entry, AliasedCostEntry))
writes: Final = itertools.chain.from_iterable(
((key, entry) for key in entry.registered_by.intersection(keys)) for entry in entries
)
return frozenset(key for key, entry in writes if cost_map.get(key) is not entry)
def model_cost_row_writers(rows: Iterable[str]) -> frozenset[str]:
"""Every ``register_model`` key whose write landed on one of ``rows``."""
entries: Final = (_builtin_cost_map().get(row) for row in rows)
@ -4184,6 +4200,52 @@ def _without_fields_left_by_earlier_writes(
displaced_values=tuple((name, value) for name, value in current.displaced_values if name not in dropped),
saved_write_fields=current.saved_write_fields,
latest_unsaved_write_fields=current.latest_unsaved_write_fields,
written_values=tuple((name, value) for name, value in current.written_values if name not in dropped),
)
def model_cost_fields_set_in_place(rows: Iterable[str]) -> Mapping[str, Mapping[str, object]]:
"""The fields of each of ``rows`` set in place, as ``litellm.model_cost[row][field] = value`` does, since the
``register_model`` write that last put a value there, read before the rows are rebuilt so the rebuild can set
them back last."""
cost_map: Final = _builtin_cost_map()
entries: Final = ((row, cost_map.get(row)) for row in rows)
set_in_place: Final = (
(row, _fields_set_in_place(entry)) for row, entry in entries if isinstance(entry, AliasedCostEntry)
)
return MappingProxyType({row: fields for row, fields in set_in_place if fields})
def _fields_set_in_place(entry: AliasedCostEntry) -> Mapping[str, object]:
fields: Final = ((name, entry.get(name), written) for name, written in entry.written_values)
return MappingProxyType({name: value for name, value, written in fields if value is not None and value != written})
def put_back_model_cost_fields_set_in_place(fields: Mapping[str, Mapping[str, object]]) -> None:
"""Set each row's ``fields`` back over what a rebuild wrote there, so the next rebuild still reads them as set in
place."""
cost_map: Final = _builtin_cost_map()
entries: Final = ((row, cost_map.get(row), values) for row, values in fields.items())
for row, entry, values in entries:
if isinstance(entry, AliasedCostEntry):
cost_map.update(
dict.fromkeys((row, *_names_sharing_entry(cost_map, row)), _with_fields_set_in_place(entry, values))
)
_invalidate_model_cost_lowercase_map()
def _with_fields_set_in_place(entry: AliasedCostEntry, values: Mapping[str, object]) -> AliasedCostEntry:
return AliasedCostEntry(
{**entry, **values},
entry.names,
registered_fields=entry.registered_fields,
created_by_registration=entry.created_by_registration,
registered_by=entry.registered_by,
written_fields=entry.written_fields,
displaced_values=entry.displaced_values,
saved_write_fields=entry.saved_write_fields,
latest_unsaved_write_fields=entry.latest_unsaved_write_fields,
written_values=entry.written_values,
)

View file

@ -2744,6 +2744,70 @@ def test_deleting_a_flagged_deployment_keeps_what_a_deployment_whose_id_names_th
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
_ID_ALIAS_BACKEND: Final = "hosted_vllm/withdraw-test-id-alias"
def _id_alias_deployment(model_id: str, **model_info: object) -> DeploymentTypedDict:
return _cache_point_deployment(model_id, model=_ID_ALIAS_BACKEND, **model_info)
def test_deleting_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None:
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
router: Final = Router(
model_list=[_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer})]
)
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is not catalog_answer
assert router.delete_deployment(id=_CACHE_POINT_DEPLOYMENT_MODEL) is not None
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
def test_editing_the_flag_off_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None:
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
router: Final = Router(
model_list=[_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer})]
)
assert router.upsert_deployment(Deployment(**_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL))) is not None
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
def test_replacing_the_model_list_without_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None:
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
other: Final = _cache_point_deployment("other", model="hosted_vllm/withdraw-test-other")
router: Final = Router(
model_list=[
other,
_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer}),
]
)
router.set_model_list([other])
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
def test_deleting_a_flagged_deployment_keeps_what_each_deployment_whose_id_resolves_to_the_row_wrote() -> None:
row: Final = _shared_cache_point_row()
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
router: Final = Router(
model_list=[
_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, withdraw_test_first_note="kept"),
_id_alias_deployment(f"bedrock/converse/{_CACHE_POINT_MODEL}", withdraw_test_second_note="kept"),
_cache_point_deployment("flagged", **{_CACHE_POINT_FLAG: not catalog_answer}),
]
)
assert litellm.model_cost[row].get("withdraw_test_first_note") == "kept"
assert router.delete_deployment(id="flagged") is not None
assert litellm.model_cost[row].get("withdraw_test_first_note") == "kept"
assert litellm.model_cost[row].get("withdraw_test_second_note") == "kept"
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
def test_deleting_a_deployment_keeps_a_limit_registered_on_a_row_the_rebuild_leaves_alone() -> None:
shared_row: Final = "azure/gpt-4o"
responses_model: Final = "azure/responses/gpt-4o"
@ -2831,6 +2895,47 @@ def test_deleting_a_flagged_deployment_keeps_a_price_an_sdk_user_set_on_the_cata
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
def test_deleting_deployments_keeps_a_limit_an_sdk_user_set_in_place_after_a_sibling_wrote_it() -> None:
row: Final = _shared_cache_point_row()
written_limit: Final = _unseen_max_output_tokens(row)
router: Final = Router(
model_list=[
_cache_point_deployment("plain"),
_cache_point_deployment("limited", max_output_tokens=written_limit),
]
)
litellm.model_cost[row]["max_output_tokens"] = written_limit + 1
flagged: Final = _cache_point_deployment(
"flagged", **{_CACHE_POINT_FLAG: not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)}
)
assert router.upsert_deployment(Deployment(**flagged)) is not None
assert router.delete_deployment(id="flagged") is not None
assert litellm.model_cost[row].get("max_output_tokens") == written_limit + 1
assert router.delete_deployment(id="plain") is not None
assert litellm.model_cost[row].get("max_output_tokens") == written_limit + 1
def test_deleting_a_flagged_deployment_gives_back_a_limit_a_sibling_wrote_over_an_earlier_in_place_edit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
row: Final = _shared_cache_point_row()
written_limit: Final = _unseen_max_output_tokens(row)
monkeypatch.setitem(litellm.model_cost[row], "max_output_tokens", written_limit + 1)
router: Final = Router(
model_list=[
_cache_point_deployment("limited", max_output_tokens=written_limit),
_cache_point_deployment(
"flagged", **{_CACHE_POINT_FLAG: not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)}
),
]
)
assert router.delete_deployment(id="flagged") is not None
assert litellm.model_cost[row].get("max_output_tokens") == written_limit
def test_deleting_a_flagged_deployment_keeps_the_limit_a_sibling_spelling_the_same_row_differently_sets() -> None:
row: Final = _shared_cache_point_row()
sibling_limit: Final = _unseen_max_output_tokens(row)
@ -2959,6 +3064,33 @@ def test_deleting_one_of_many_differently_priced_deployments_rewrites_the_shared
assert litellm.model_cost["sibling-7"].get("input_cost_per_token") == pytest.approx(8e-7)
def test_deleting_one_of_many_deployments_alternating_two_flags_rewrites_the_shared_row_once_per_flag() -> None:
row: Final = _shared_cache_point_row()
flag: Final = not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
host: Final = _CostMapWriteRecordingHost()
router: Final = Router.from_config(
build_router_config(
router_arguments(
model_list=[
*(
_cache_point_deployment(f"sibling-{n}", **{_CACHE_POINT_FLAG: (flag, not flag)[n % 2]})
for n in range(51)
),
_cache_point_deployment("doomed", max_output_tokens=_unseen_max_output_tokens(row)),
]
)
),
host=host,
)
host.cost_map_writes.reset_mock()
assert router.delete_deployment(id="doomed") is not None
assert host.cost_map_writes.call_count == 2
assert litellm.model_cost[row].get(_CACHE_POINT_FLAG) is flag
assert litellm.model_cost[row].get("max_output_tokens") == _catalog_field(row, "max_output_tokens")
@pytest.mark.parametrize(
("registered_before_the_router", "deleted", "saved_flag_wins"),
[(True, "plain", False), (False, "plain", True), (True, "flagged", True)],
@ -3006,9 +3138,7 @@ def test_editing_a_priced_deployment_keeps_its_price_on_its_row_throughout_the_e
host: Final = _IdRowPriceRecordingHost("edited")
router: Final = Router.from_config(
build_router_config(
router_arguments(
model_list=[_priced_cache_point_deployment("edited", 3e-6, **{_CACHE_POINT_FLAG: True})]
)
router_arguments(model_list=[_priced_cache_point_deployment("edited", 3e-6, **{_CACHE_POINT_FLAG: True})])
),
host=host,
)