mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): replay each distinct cost write once and keep in-place edits through a rebuild
This commit is contained in:
parent
8a9a8a5f71
commit
979d01db6c
4 changed files with 256 additions and 30 deletions
|
|
@ -570,6 +570,7 @@ class AliasedCostEntry(dict[str, object]):
|
|||
"registered_fields",
|
||||
"saved_write_fields",
|
||||
"written_fields",
|
||||
"written_values",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
|
|
@ -583,6 +584,7 @@ class AliasedCostEntry(dict[str, object]):
|
|||
displaced_values: tuple[tuple[str, object], ...] = (),
|
||||
saved_write_fields: frozenset[str] = frozenset(),
|
||||
latest_unsaved_write_fields: frozenset[str] = frozenset(),
|
||||
written_values: tuple[tuple[str, object], ...] = (),
|
||||
) -> None:
|
||||
super().__init__(entry)
|
||||
self.names: tuple[str, ...] = names
|
||||
|
|
@ -593,6 +595,7 @@ class AliasedCostEntry(dict[str, object]):
|
|||
self.displaced_values: tuple[tuple[str, object], ...] = displaced_values
|
||||
self.saved_write_fields: frozenset[str] = saved_write_fields
|
||||
self.latest_unsaved_write_fields: frozenset[str] = latest_unsaved_write_fields
|
||||
self.written_values: tuple[tuple[str, object], ...] = written_values
|
||||
|
||||
|
||||
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
|
||||
|
|
|
|||
|
|
@ -517,11 +517,14 @@ from litellm.utils import (
|
|||
is_generalized_model_info,
|
||||
is_region_allowed,
|
||||
lookup_model_info_for_registration,
|
||||
model_cost_fields_set_in_place,
|
||||
model_cost_keys_written_onto_other_rows,
|
||||
model_cost_registration_key,
|
||||
model_cost_row_writers,
|
||||
model_cost_rows_written_by,
|
||||
model_info_with_pricing_overrides,
|
||||
provider_rejectable_params,
|
||||
put_back_model_cost_fields_set_in_place,
|
||||
restore_model_cost_rows_to_catalog,
|
||||
runtime_model_cost_registrations_on,
|
||||
runtime_model_cost_registrations_written_last,
|
||||
|
|
@ -1138,10 +1141,14 @@ class _SharedCostWrite:
|
|||
def lands_on(self, writers: frozenset[str]) -> bool:
|
||||
return self.model_id in writers or not writers.isdisjoint(self.backend_keys)
|
||||
|
||||
def replay_key(self, stale: frozenset[str]) -> tuple[str | None, str]:
|
||||
"""Equal for two entries only when replaying both writes the ``stale`` rows exactly as replaying one does. A
|
||||
shared row never takes a price, only what a price being set pulls in from the backend model, so prices count
|
||||
only as set or unset, and a stale id row keeps its write apart by its id."""
|
||||
@property
|
||||
def sets_mode(self) -> bool:
|
||||
return isinstance(self.model_info.get("mode"), str)
|
||||
|
||||
def replay_key(self, targets: frozenset[str]) -> tuple[str | None, str]:
|
||||
"""Equal for two entries only when replaying both writes the ``targets`` rows exactly as replaying one does.
|
||||
A shared row never takes a price, only what a price being set pulls in from the backend model, so prices count
|
||||
only as set or unset, and an id that names a target row or wrote onto one keeps its write apart by its id."""
|
||||
write_inputs: Final = (
|
||||
self.model,
|
||||
self.custom_llm_provider,
|
||||
|
|
@ -1149,7 +1156,7 @@ class _SharedCostWrite:
|
|||
_SharedCostWrite._write_inputs(self.litellm_params, _LITELLM_PARAMS_COST_WRITE_INPUTS),
|
||||
)
|
||||
return (
|
||||
self.model_id if self.model_id in stale else None,
|
||||
self.model_id if self.model_id in targets else None,
|
||||
json.dumps(write_inputs, sort_keys=True, default=repr),
|
||||
)
|
||||
|
||||
|
|
@ -12128,7 +12135,11 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# OLD model_name/tags) before the re-add below re-registers it.
|
||||
self._unregister_pre_routing_strategy_for_deployment(deployment=_deployment_on_router)
|
||||
self._rebuild_model_cost_rows(
|
||||
lambda: self._rebuild_cost_rows_written_by(PythonCore._backend_write_keys(_deployment_on_router))
|
||||
lambda: self._rebuild_cost_rows_written_by(
|
||||
PythonCore._shared_write_keys(
|
||||
(PythonCore._deployment_shared_cost_write(_deployment_on_router),)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# if the model_id is not in router
|
||||
|
|
@ -12265,7 +12276,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
|
||||
Takes only a ``Deployment``, so it registers pricing for a deployment that
|
||||
is never added to ``self._core_model_list`` (a per-request client-side-credential
|
||||
deployment) just as readily as one that is. ``onto`` limits the writes to those rows.
|
||||
deployment) just as readily as one that is. ``onto`` limits the writes to those rows, and the id's write to an
|
||||
id it names, which is how a rebuild replays an id whose write landed on a rebuilt row named otherwise.
|
||||
"""
|
||||
PythonCore._register_deployment_in_model_cost(
|
||||
model_id=deployment.model_info.id,
|
||||
|
|
@ -12391,13 +12403,14 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _backend_write_keys(deployment: Deployment) -> frozenset[str]:
|
||||
return frozenset(
|
||||
PythonCore._backend_cost_map_keys(
|
||||
model=deployment.litellm_params.model,
|
||||
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
|
||||
)
|
||||
)
|
||||
def _shared_write_keys(writes: Iterable[_SharedCostWrite]) -> frozenset[str]:
|
||||
"""The ``register_model`` keys of ``writes`` whose rows other deployments read too: every backend key, and
|
||||
each id whose write landed on a row named otherwise, such as the ``gpt-4o`` row an ``openai/gpt-4o`` id
|
||||
resolves to."""
|
||||
listed: Final = tuple(writes)
|
||||
ids: Final = frozenset(write.model_id for write in listed if write.model_id is not None)
|
||||
backend_keys: Final = chain.from_iterable(write.backend_keys for write in listed)
|
||||
return frozenset(backend_keys) | model_cost_keys_written_onto_other_rows(ids)
|
||||
|
||||
def _rebuild_rows_a_replaced_model_list_wrote(self, previous: Iterable[object]) -> None:
|
||||
"""Rebuild the shared rows a replaced ``model_list`` wrote and trim the id rows of the deployments still
|
||||
|
|
@ -12406,7 +12419,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
writes: Final = tuple(
|
||||
write for write in map(PythonCore._entry_shared_cost_write, previous) if write is not None
|
||||
)
|
||||
self._rebuild_cost_rows_written_by(frozenset(chain.from_iterable(write.backend_keys for write in writes)))
|
||||
self._rebuild_cost_rows_written_by(PythonCore._shared_write_keys(writes))
|
||||
self._give_back_id_row_fields(
|
||||
write.model_id
|
||||
for write in writes
|
||||
|
|
@ -12458,39 +12471,55 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
def _rebuild_cost_rows_written_by(self, written_keys: frozenset[str]) -> None:
|
||||
"""Rebuild the rows ``register_model`` writes of ``written_keys`` landed on: the loaded catalog row, then the
|
||||
``register_model`` registrations replayed on every reload, then every live deployment that wrote it in
|
||||
``model_list`` order, then again the fields of those registrations no deployment wrote over since, so each
|
||||
field keeps the order its writes came in, whether the registration came before the router or after it. The
|
||||
replay writes only those rows, so a row it leaves alone keeps what later deployments and registrations put
|
||||
there, and a run of deployments with the same write replays it once, which leaves the rows as replaying
|
||||
every one of them does."""
|
||||
``model_list`` order, then again the fields of those registrations no deployment wrote over since, then the
|
||||
fields set in place after their last write, so each field keeps the order its writes came in, whether the
|
||||
registration came before the router or after it. The replay writes only those rows, so a row it leaves alone
|
||||
keeps what later deployments and registrations put there, and deployments with the same write replay it once,
|
||||
which leaves the rows as replaying every one of them does."""
|
||||
stale: Final = model_cost_rows_written_by(written_keys) if written_keys else frozenset[str]()
|
||||
if not stale:
|
||||
return
|
||||
writers: Final = model_cost_row_writers(stale)
|
||||
targets: Final = stale | writers
|
||||
routers: Final = tuple(_live_routers)
|
||||
entries: Final = tuple(chain.from_iterable(router._model_list_entries() for router in routers))
|
||||
writes: Final = (PythonCore._entry_shared_cost_write(entry) for entry in entries)
|
||||
landing: Final = tuple(
|
||||
(position, write.replay_key(stale))
|
||||
for position, write in enumerate(writes)
|
||||
if write is not None and write.lands_on(writers)
|
||||
(position, write) for position, write in enumerate(writes) if write is not None and write.lands_on(writers)
|
||||
)
|
||||
replay_runs: Final = groupby(landing, key=itemgetter(1))
|
||||
replayed: Final = tuple(
|
||||
PythonCore._buildable_deployments(entries[tuple(run)[-1][0]] for _replay_key, run in replay_runs)
|
||||
PythonCore._buildable_deployments(
|
||||
entries[position] for position in PythonCore._replayed_positions(landing, targets)
|
||||
)
|
||||
)
|
||||
persisted: Final = runtime_model_cost_registrations_on(stale)
|
||||
written_last: Final = runtime_model_cost_registrations_written_last(persisted)
|
||||
set_in_place: Final = model_cost_fields_set_in_place(stale)
|
||||
restore_model_cost_rows_to_catalog(stale)
|
||||
if persisted:
|
||||
self._host.invoke("register_model")(model_cost=dict(persisted))
|
||||
for deployment in replayed:
|
||||
PythonCore._register_deployment_pricing(deployment, onto=stale, host=self._host)
|
||||
PythonCore._register_deployment_pricing(deployment, onto=targets, host=self._host)
|
||||
if written_last:
|
||||
self._host.invoke("register_model")(model_cost=dict(written_last))
|
||||
if set_in_place:
|
||||
put_back_model_cost_fields_set_in_place(set_in_place)
|
||||
for router in routers:
|
||||
router._invalidate_model_group_info_cache()
|
||||
|
||||
@staticmethod
|
||||
def _replayed_positions(
|
||||
landing: tuple[tuple[int, _SharedCostWrite], ...], targets: frozenset[str]
|
||||
) -> tuple[int, ...]:
|
||||
"""The position of the last of each set of ``landing`` writes that leave the ``targets`` rows as one of them
|
||||
does. Each field keeps the last value written to it, so only the last write of each set counts, unless a write
|
||||
sets ``mode``: what a ``mode`` write does depends on the mode the row holds, so then only back-to-back runs
|
||||
collapse."""
|
||||
keyed: Final = tuple((position, write.replay_key(targets)) for position, write in landing)
|
||||
if any(write.sets_mode for _position, write in landing):
|
||||
return tuple(tuple(run)[-1][0] for _replay_key, run in groupby(keyed, key=itemgetter(1)))
|
||||
return tuple(sorted({replay_key: position for position, replay_key in keyed}.values()))
|
||||
|
||||
def _core_delete_deployment(self, id: str) -> Deployment | None:
|
||||
"""
|
||||
Parameters:
|
||||
|
|
@ -12526,7 +12555,9 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
id,
|
||||
)
|
||||
self._rebuild_model_cost_rows(
|
||||
lambda: self._rebuild_cost_rows_written_by(PythonCore._backend_write_keys(removed_deployment()))
|
||||
lambda: self._rebuild_cost_rows_written_by(
|
||||
PythonCore._shared_write_keys((PythonCore._deployment_shared_cost_write(removed_deployment()),))
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -3810,6 +3810,7 @@ def _replace_cost_map_entry(
|
|||
}.items()
|
||||
)
|
||||
created_by_registration: Final = previous.created_by_registration if current is previous else current is None
|
||||
written_now: Final = {name: value for name, value in entry.items() if name in changed_fields or name in set_fields}
|
||||
cost_map.update(
|
||||
dict.fromkeys(
|
||||
(key, *_names_sharing_entry(cost_map, key)),
|
||||
|
|
@ -3825,6 +3826,10 @@ def _replace_cost_map_entry(
|
|||
previous.saved_write_fields | set_fields if saved else previous.saved_write_fields - set_fields
|
||||
),
|
||||
latest_unsaved_write_fields=previous.latest_unsaved_write_fields if saved else set_fields,
|
||||
written_values=(
|
||||
*((name, value) for name, value in previous.written_values if name not in written_now),
|
||||
*written_now.items(),
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
|
@ -4110,6 +4115,17 @@ def model_cost_rows_written_by(keys: Collection[str]) -> frozenset[str]:
|
|||
)
|
||||
|
||||
|
||||
def model_cost_keys_written_onto_other_rows(keys: Collection[str]) -> frozenset[str]:
|
||||
"""Each of ``keys`` whose ``register_model`` write landed on a row named otherwise, as an ``openai/gpt-4o``
|
||||
write lands on the ``gpt-4o`` row."""
|
||||
cost_map: Final = _builtin_cost_map()
|
||||
entries: Final = (entry for entry in tuple(cost_map.values()) if isinstance(entry, AliasedCostEntry))
|
||||
writes: Final = itertools.chain.from_iterable(
|
||||
((key, entry) for key in entry.registered_by.intersection(keys)) for entry in entries
|
||||
)
|
||||
return frozenset(key for key, entry in writes if cost_map.get(key) is not entry)
|
||||
|
||||
|
||||
def model_cost_row_writers(rows: Iterable[str]) -> frozenset[str]:
|
||||
"""Every ``register_model`` key whose write landed on one of ``rows``."""
|
||||
entries: Final = (_builtin_cost_map().get(row) for row in rows)
|
||||
|
|
@ -4184,6 +4200,52 @@ def _without_fields_left_by_earlier_writes(
|
|||
displaced_values=tuple((name, value) for name, value in current.displaced_values if name not in dropped),
|
||||
saved_write_fields=current.saved_write_fields,
|
||||
latest_unsaved_write_fields=current.latest_unsaved_write_fields,
|
||||
written_values=tuple((name, value) for name, value in current.written_values if name not in dropped),
|
||||
)
|
||||
|
||||
|
||||
def model_cost_fields_set_in_place(rows: Iterable[str]) -> Mapping[str, Mapping[str, object]]:
|
||||
"""The fields of each of ``rows`` set in place, as ``litellm.model_cost[row][field] = value`` does, since the
|
||||
``register_model`` write that last put a value there, read before the rows are rebuilt so the rebuild can set
|
||||
them back last."""
|
||||
cost_map: Final = _builtin_cost_map()
|
||||
entries: Final = ((row, cost_map.get(row)) for row in rows)
|
||||
set_in_place: Final = (
|
||||
(row, _fields_set_in_place(entry)) for row, entry in entries if isinstance(entry, AliasedCostEntry)
|
||||
)
|
||||
return MappingProxyType({row: fields for row, fields in set_in_place if fields})
|
||||
|
||||
|
||||
def _fields_set_in_place(entry: AliasedCostEntry) -> Mapping[str, object]:
|
||||
fields: Final = ((name, entry.get(name), written) for name, written in entry.written_values)
|
||||
return MappingProxyType({name: value for name, value, written in fields if value is not None and value != written})
|
||||
|
||||
|
||||
def put_back_model_cost_fields_set_in_place(fields: Mapping[str, Mapping[str, object]]) -> None:
|
||||
"""Set each row's ``fields`` back over what a rebuild wrote there, so the next rebuild still reads them as set in
|
||||
place."""
|
||||
cost_map: Final = _builtin_cost_map()
|
||||
entries: Final = ((row, cost_map.get(row), values) for row, values in fields.items())
|
||||
for row, entry, values in entries:
|
||||
if isinstance(entry, AliasedCostEntry):
|
||||
cost_map.update(
|
||||
dict.fromkeys((row, *_names_sharing_entry(cost_map, row)), _with_fields_set_in_place(entry, values))
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def _with_fields_set_in_place(entry: AliasedCostEntry, values: Mapping[str, object]) -> AliasedCostEntry:
|
||||
return AliasedCostEntry(
|
||||
{**entry, **values},
|
||||
entry.names,
|
||||
registered_fields=entry.registered_fields,
|
||||
created_by_registration=entry.created_by_registration,
|
||||
registered_by=entry.registered_by,
|
||||
written_fields=entry.written_fields,
|
||||
displaced_values=entry.displaced_values,
|
||||
saved_write_fields=entry.saved_write_fields,
|
||||
latest_unsaved_write_fields=entry.latest_unsaved_write_fields,
|
||||
written_values=entry.written_values,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2744,6 +2744,70 @@ def test_deleting_a_flagged_deployment_keeps_what_a_deployment_whose_id_names_th
|
|||
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
|
||||
|
||||
|
||||
_ID_ALIAS_BACKEND: Final = "hosted_vllm/withdraw-test-id-alias"
|
||||
|
||||
|
||||
def _id_alias_deployment(model_id: str, **model_info: object) -> DeploymentTypedDict:
|
||||
return _cache_point_deployment(model_id, model=_ID_ALIAS_BACKEND, **model_info)
|
||||
|
||||
|
||||
def test_deleting_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None:
|
||||
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
|
||||
router: Final = Router(
|
||||
model_list=[_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer})]
|
||||
)
|
||||
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is not catalog_answer
|
||||
|
||||
assert router.delete_deployment(id=_CACHE_POINT_DEPLOYMENT_MODEL) is not None
|
||||
|
||||
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
|
||||
|
||||
|
||||
def test_editing_the_flag_off_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None:
|
||||
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
|
||||
router: Final = Router(
|
||||
model_list=[_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer})]
|
||||
)
|
||||
|
||||
assert router.upsert_deployment(Deployment(**_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL))) is not None
|
||||
|
||||
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
|
||||
|
||||
|
||||
def test_replacing_the_model_list_without_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None:
|
||||
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
|
||||
other: Final = _cache_point_deployment("other", model="hosted_vllm/withdraw-test-other")
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
other,
|
||||
_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer}),
|
||||
]
|
||||
)
|
||||
|
||||
router.set_model_list([other])
|
||||
|
||||
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
|
||||
|
||||
|
||||
def test_deleting_a_flagged_deployment_keeps_what_each_deployment_whose_id_resolves_to_the_row_wrote() -> None:
|
||||
row: Final = _shared_cache_point_row()
|
||||
catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, withdraw_test_first_note="kept"),
|
||||
_id_alias_deployment(f"bedrock/converse/{_CACHE_POINT_MODEL}", withdraw_test_second_note="kept"),
|
||||
_cache_point_deployment("flagged", **{_CACHE_POINT_FLAG: not catalog_answer}),
|
||||
]
|
||||
)
|
||||
assert litellm.model_cost[row].get("withdraw_test_first_note") == "kept"
|
||||
|
||||
assert router.delete_deployment(id="flagged") is not None
|
||||
|
||||
assert litellm.model_cost[row].get("withdraw_test_first_note") == "kept"
|
||||
assert litellm.model_cost[row].get("withdraw_test_second_note") == "kept"
|
||||
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
|
||||
|
||||
|
||||
def test_deleting_a_deployment_keeps_a_limit_registered_on_a_row_the_rebuild_leaves_alone() -> None:
|
||||
shared_row: Final = "azure/gpt-4o"
|
||||
responses_model: Final = "azure/responses/gpt-4o"
|
||||
|
|
@ -2831,6 +2895,47 @@ def test_deleting_a_flagged_deployment_keeps_a_price_an_sdk_user_set_on_the_cata
|
|||
assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer
|
||||
|
||||
|
||||
def test_deleting_deployments_keeps_a_limit_an_sdk_user_set_in_place_after_a_sibling_wrote_it() -> None:
|
||||
row: Final = _shared_cache_point_row()
|
||||
written_limit: Final = _unseen_max_output_tokens(row)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
_cache_point_deployment("plain"),
|
||||
_cache_point_deployment("limited", max_output_tokens=written_limit),
|
||||
]
|
||||
)
|
||||
litellm.model_cost[row]["max_output_tokens"] = written_limit + 1
|
||||
flagged: Final = _cache_point_deployment(
|
||||
"flagged", **{_CACHE_POINT_FLAG: not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)}
|
||||
)
|
||||
assert router.upsert_deployment(Deployment(**flagged)) is not None
|
||||
|
||||
assert router.delete_deployment(id="flagged") is not None
|
||||
assert litellm.model_cost[row].get("max_output_tokens") == written_limit + 1
|
||||
assert router.delete_deployment(id="plain") is not None
|
||||
assert litellm.model_cost[row].get("max_output_tokens") == written_limit + 1
|
||||
|
||||
|
||||
def test_deleting_a_flagged_deployment_gives_back_a_limit_a_sibling_wrote_over_an_earlier_in_place_edit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
row: Final = _shared_cache_point_row()
|
||||
written_limit: Final = _unseen_max_output_tokens(row)
|
||||
monkeypatch.setitem(litellm.model_cost[row], "max_output_tokens", written_limit + 1)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
_cache_point_deployment("limited", max_output_tokens=written_limit),
|
||||
_cache_point_deployment(
|
||||
"flagged", **{_CACHE_POINT_FLAG: not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)}
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
assert router.delete_deployment(id="flagged") is not None
|
||||
|
||||
assert litellm.model_cost[row].get("max_output_tokens") == written_limit
|
||||
|
||||
|
||||
def test_deleting_a_flagged_deployment_keeps_the_limit_a_sibling_spelling_the_same_row_differently_sets() -> None:
|
||||
row: Final = _shared_cache_point_row()
|
||||
sibling_limit: Final = _unseen_max_output_tokens(row)
|
||||
|
|
@ -2959,6 +3064,33 @@ def test_deleting_one_of_many_differently_priced_deployments_rewrites_the_shared
|
|||
assert litellm.model_cost["sibling-7"].get("input_cost_per_token") == pytest.approx(8e-7)
|
||||
|
||||
|
||||
def test_deleting_one_of_many_deployments_alternating_two_flags_rewrites_the_shared_row_once_per_flag() -> None:
|
||||
row: Final = _shared_cache_point_row()
|
||||
flag: Final = not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)
|
||||
host: Final = _CostMapWriteRecordingHost()
|
||||
router: Final = Router.from_config(
|
||||
build_router_config(
|
||||
router_arguments(
|
||||
model_list=[
|
||||
*(
|
||||
_cache_point_deployment(f"sibling-{n}", **{_CACHE_POINT_FLAG: (flag, not flag)[n % 2]})
|
||||
for n in range(51)
|
||||
),
|
||||
_cache_point_deployment("doomed", max_output_tokens=_unseen_max_output_tokens(row)),
|
||||
]
|
||||
)
|
||||
),
|
||||
host=host,
|
||||
)
|
||||
host.cost_map_writes.reset_mock()
|
||||
|
||||
assert router.delete_deployment(id="doomed") is not None
|
||||
|
||||
assert host.cost_map_writes.call_count == 2
|
||||
assert litellm.model_cost[row].get(_CACHE_POINT_FLAG) is flag
|
||||
assert litellm.model_cost[row].get("max_output_tokens") == _catalog_field(row, "max_output_tokens")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("registered_before_the_router", "deleted", "saved_flag_wins"),
|
||||
[(True, "plain", False), (False, "plain", True), (True, "flagged", True)],
|
||||
|
|
@ -3006,9 +3138,7 @@ def test_editing_a_priced_deployment_keeps_its_price_on_its_row_throughout_the_e
|
|||
host: Final = _IdRowPriceRecordingHost("edited")
|
||||
router: Final = Router.from_config(
|
||||
build_router_config(
|
||||
router_arguments(
|
||||
model_list=[_priced_cache_point_deployment("edited", 3e-6, **{_CACHE_POINT_FLAG: True})]
|
||||
)
|
||||
router_arguments(model_list=[_priced_cache_point_deployment("edited", 3e-6, **{_CACHE_POINT_FLAG: True})])
|
||||
),
|
||||
host=host,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue