diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index a82faa2e49b..826726d8102 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -570,6 +570,7 @@ class AliasedCostEntry(dict[str, object]): "registered_fields", "saved_write_fields", "written_fields", + "written_values", ) def __init__( @@ -583,6 +584,7 @@ class AliasedCostEntry(dict[str, object]): displaced_values: tuple[tuple[str, object], ...] = (), saved_write_fields: frozenset[str] = frozenset(), latest_unsaved_write_fields: frozenset[str] = frozenset(), + written_values: tuple[tuple[str, object], ...] = (), ) -> None: super().__init__(entry) self.names: tuple[str, ...] = names @@ -593,6 +595,7 @@ class AliasedCostEntry(dict[str, object]): self.displaced_values: tuple[tuple[str, object], ...] = displaced_values self.saved_write_fields: frozenset[str] = saved_write_fields self.latest_unsaved_write_fields: frozenset[str] = latest_unsaved_write_fields + self.written_values: tuple[tuple[str, object], ...] = written_values def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]: diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 6f87ea5baab..24b517bfc8e 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -517,11 +517,14 @@ from litellm.utils import ( is_generalized_model_info, is_region_allowed, lookup_model_info_for_registration, + model_cost_fields_set_in_place, + model_cost_keys_written_onto_other_rows, model_cost_registration_key, model_cost_row_writers, model_cost_rows_written_by, model_info_with_pricing_overrides, provider_rejectable_params, + put_back_model_cost_fields_set_in_place, restore_model_cost_rows_to_catalog, runtime_model_cost_registrations_on, runtime_model_cost_registrations_written_last, @@ -1138,10 +1141,14 @@ class _SharedCostWrite: def lands_on(self, writers: frozenset[str]) -> bool: return self.model_id in writers or not writers.isdisjoint(self.backend_keys) - def replay_key(self, stale: frozenset[str]) -> tuple[str | None, str]: - """Equal for two entries only when replaying both writes the ``stale`` rows exactly as replaying one does. A - shared row never takes a price, only what a price being set pulls in from the backend model, so prices count - only as set or unset, and a stale id row keeps its write apart by its id.""" + @property + def sets_mode(self) -> bool: + return isinstance(self.model_info.get("mode"), str) + + def replay_key(self, targets: frozenset[str]) -> tuple[str | None, str]: + """Equal for two entries only when replaying both writes the ``targets`` rows exactly as replaying one does. + A shared row never takes a price, only what a price being set pulls in from the backend model, so prices count + only as set or unset, and an id that names a target row or wrote onto one keeps its write apart by its id.""" write_inputs: Final = ( self.model, self.custom_llm_provider, @@ -1149,7 +1156,7 @@ class _SharedCostWrite: _SharedCostWrite._write_inputs(self.litellm_params, _LITELLM_PARAMS_COST_WRITE_INPUTS), ) return ( - self.model_id if self.model_id in stale else None, + self.model_id if self.model_id in targets else None, json.dumps(write_inputs, sort_keys=True, default=repr), ) @@ -12128,7 +12135,11 @@ class PythonCore(DeprecatedRouterMembers): # OLD model_name/tags) before the re-add below re-registers it. self._unregister_pre_routing_strategy_for_deployment(deployment=_deployment_on_router) self._rebuild_model_cost_rows( - lambda: self._rebuild_cost_rows_written_by(PythonCore._backend_write_keys(_deployment_on_router)) + lambda: self._rebuild_cost_rows_written_by( + PythonCore._shared_write_keys( + (PythonCore._deployment_shared_cost_write(_deployment_on_router),) + ) + ) ) # if the model_id is not in router @@ -12265,7 +12276,8 @@ class PythonCore(DeprecatedRouterMembers): Takes only a ``Deployment``, so it registers pricing for a deployment that is never added to ``self._core_model_list`` (a per-request client-side-credential - deployment) just as readily as one that is. ``onto`` limits the writes to those rows. + deployment) just as readily as one that is. ``onto`` limits the writes to those rows, and the id's write to an + id it names, which is how a rebuild replays an id whose write landed on a rebuilt row named otherwise. """ PythonCore._register_deployment_in_model_cost( model_id=deployment.model_info.id, @@ -12391,13 +12403,14 @@ class PythonCore(DeprecatedRouterMembers): ) @staticmethod - def _backend_write_keys(deployment: Deployment) -> frozenset[str]: - return frozenset( - PythonCore._backend_cost_map_keys( - model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - ) + def _shared_write_keys(writes: Iterable[_SharedCostWrite]) -> frozenset[str]: + """The ``register_model`` keys of ``writes`` whose rows other deployments read too: every backend key, and + each id whose write landed on a row named otherwise, such as the ``gpt-4o`` row an ``openai/gpt-4o`` id + resolves to.""" + listed: Final = tuple(writes) + ids: Final = frozenset(write.model_id for write in listed if write.model_id is not None) + backend_keys: Final = chain.from_iterable(write.backend_keys for write in listed) + return frozenset(backend_keys) | model_cost_keys_written_onto_other_rows(ids) def _rebuild_rows_a_replaced_model_list_wrote(self, previous: Iterable[object]) -> None: """Rebuild the shared rows a replaced ``model_list`` wrote and trim the id rows of the deployments still @@ -12406,7 +12419,7 @@ class PythonCore(DeprecatedRouterMembers): writes: Final = tuple( write for write in map(PythonCore._entry_shared_cost_write, previous) if write is not None ) - self._rebuild_cost_rows_written_by(frozenset(chain.from_iterable(write.backend_keys for write in writes))) + self._rebuild_cost_rows_written_by(PythonCore._shared_write_keys(writes)) self._give_back_id_row_fields( write.model_id for write in writes @@ -12458,39 +12471,55 @@ class PythonCore(DeprecatedRouterMembers): def _rebuild_cost_rows_written_by(self, written_keys: frozenset[str]) -> None: """Rebuild the rows ``register_model`` writes of ``written_keys`` landed on: the loaded catalog row, then the ``register_model`` registrations replayed on every reload, then every live deployment that wrote it in - ``model_list`` order, then again the fields of those registrations no deployment wrote over since, so each - field keeps the order its writes came in, whether the registration came before the router or after it. The - replay writes only those rows, so a row it leaves alone keeps what later deployments and registrations put - there, and a run of deployments with the same write replays it once, which leaves the rows as replaying - every one of them does.""" + ``model_list`` order, then again the fields of those registrations no deployment wrote over since, then the + fields set in place after their last write, so each field keeps the order its writes came in, whether the + registration came before the router or after it. The replay writes only those rows, so a row it leaves alone + keeps what later deployments and registrations put there, and deployments with the same write replay it once, + which leaves the rows as replaying every one of them does.""" stale: Final = model_cost_rows_written_by(written_keys) if written_keys else frozenset[str]() if not stale: return writers: Final = model_cost_row_writers(stale) + targets: Final = stale | writers routers: Final = tuple(_live_routers) entries: Final = tuple(chain.from_iterable(router._model_list_entries() for router in routers)) writes: Final = (PythonCore._entry_shared_cost_write(entry) for entry in entries) landing: Final = tuple( - (position, write.replay_key(stale)) - for position, write in enumerate(writes) - if write is not None and write.lands_on(writers) + (position, write) for position, write in enumerate(writes) if write is not None and write.lands_on(writers) ) - replay_runs: Final = groupby(landing, key=itemgetter(1)) replayed: Final = tuple( - PythonCore._buildable_deployments(entries[tuple(run)[-1][0]] for _replay_key, run in replay_runs) + PythonCore._buildable_deployments( + entries[position] for position in PythonCore._replayed_positions(landing, targets) + ) ) persisted: Final = runtime_model_cost_registrations_on(stale) written_last: Final = runtime_model_cost_registrations_written_last(persisted) + set_in_place: Final = model_cost_fields_set_in_place(stale) restore_model_cost_rows_to_catalog(stale) if persisted: self._host.invoke("register_model")(model_cost=dict(persisted)) for deployment in replayed: - PythonCore._register_deployment_pricing(deployment, onto=stale, host=self._host) + PythonCore._register_deployment_pricing(deployment, onto=targets, host=self._host) if written_last: self._host.invoke("register_model")(model_cost=dict(written_last)) + if set_in_place: + put_back_model_cost_fields_set_in_place(set_in_place) for router in routers: router._invalidate_model_group_info_cache() + @staticmethod + def _replayed_positions( + landing: tuple[tuple[int, _SharedCostWrite], ...], targets: frozenset[str] + ) -> tuple[int, ...]: + """The position of the last of each set of ``landing`` writes that leave the ``targets`` rows as one of them + does. Each field keeps the last value written to it, so only the last write of each set counts, unless a write + sets ``mode``: what a ``mode`` write does depends on the mode the row holds, so then only back-to-back runs + collapse.""" + keyed: Final = tuple((position, write.replay_key(targets)) for position, write in landing) + if any(write.sets_mode for _position, write in landing): + return tuple(tuple(run)[-1][0] for _replay_key, run in groupby(keyed, key=itemgetter(1))) + return tuple(sorted({replay_key: position for position, replay_key in keyed}.values())) + def _core_delete_deployment(self, id: str) -> Deployment | None: """ Parameters: @@ -12526,7 +12555,9 @@ class PythonCore(DeprecatedRouterMembers): id, ) self._rebuild_model_cost_rows( - lambda: self._rebuild_cost_rows_written_by(PythonCore._backend_write_keys(removed_deployment())) + lambda: self._rebuild_cost_rows_written_by( + PythonCore._shared_write_keys((PythonCore._deployment_shared_cost_write(removed_deployment()),)) + ) ) except Exception: return None diff --git a/litellm/utils.py b/litellm/utils.py index b03af20be01..97026e5b3c7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3810,6 +3810,7 @@ def _replace_cost_map_entry( }.items() ) created_by_registration: Final = previous.created_by_registration if current is previous else current is None + written_now: Final = {name: value for name, value in entry.items() if name in changed_fields or name in set_fields} cost_map.update( dict.fromkeys( (key, *_names_sharing_entry(cost_map, key)), @@ -3825,6 +3826,10 @@ def _replace_cost_map_entry( previous.saved_write_fields | set_fields if saved else previous.saved_write_fields - set_fields ), latest_unsaved_write_fields=previous.latest_unsaved_write_fields if saved else set_fields, + written_values=( + *((name, value) for name, value in previous.written_values if name not in written_now), + *written_now.items(), + ), ), ) ) @@ -4110,6 +4115,17 @@ def model_cost_rows_written_by(keys: Collection[str]) -> frozenset[str]: ) +def model_cost_keys_written_onto_other_rows(keys: Collection[str]) -> frozenset[str]: + """Each of ``keys`` whose ``register_model`` write landed on a row named otherwise, as an ``openai/gpt-4o`` + write lands on the ``gpt-4o`` row.""" + cost_map: Final = _builtin_cost_map() + entries: Final = (entry for entry in tuple(cost_map.values()) if isinstance(entry, AliasedCostEntry)) + writes: Final = itertools.chain.from_iterable( + ((key, entry) for key in entry.registered_by.intersection(keys)) for entry in entries + ) + return frozenset(key for key, entry in writes if cost_map.get(key) is not entry) + + def model_cost_row_writers(rows: Iterable[str]) -> frozenset[str]: """Every ``register_model`` key whose write landed on one of ``rows``.""" entries: Final = (_builtin_cost_map().get(row) for row in rows) @@ -4184,6 +4200,52 @@ def _without_fields_left_by_earlier_writes( displaced_values=tuple((name, value) for name, value in current.displaced_values if name not in dropped), saved_write_fields=current.saved_write_fields, latest_unsaved_write_fields=current.latest_unsaved_write_fields, + written_values=tuple((name, value) for name, value in current.written_values if name not in dropped), + ) + + +def model_cost_fields_set_in_place(rows: Iterable[str]) -> Mapping[str, Mapping[str, object]]: + """The fields of each of ``rows`` set in place, as ``litellm.model_cost[row][field] = value`` does, since the + ``register_model`` write that last put a value there, read before the rows are rebuilt so the rebuild can set + them back last.""" + cost_map: Final = _builtin_cost_map() + entries: Final = ((row, cost_map.get(row)) for row in rows) + set_in_place: Final = ( + (row, _fields_set_in_place(entry)) for row, entry in entries if isinstance(entry, AliasedCostEntry) + ) + return MappingProxyType({row: fields for row, fields in set_in_place if fields}) + + +def _fields_set_in_place(entry: AliasedCostEntry) -> Mapping[str, object]: + fields: Final = ((name, entry.get(name), written) for name, written in entry.written_values) + return MappingProxyType({name: value for name, value, written in fields if value is not None and value != written}) + + +def put_back_model_cost_fields_set_in_place(fields: Mapping[str, Mapping[str, object]]) -> None: + """Set each row's ``fields`` back over what a rebuild wrote there, so the next rebuild still reads them as set in + place.""" + cost_map: Final = _builtin_cost_map() + entries: Final = ((row, cost_map.get(row), values) for row, values in fields.items()) + for row, entry, values in entries: + if isinstance(entry, AliasedCostEntry): + cost_map.update( + dict.fromkeys((row, *_names_sharing_entry(cost_map, row)), _with_fields_set_in_place(entry, values)) + ) + _invalidate_model_cost_lowercase_map() + + +def _with_fields_set_in_place(entry: AliasedCostEntry, values: Mapping[str, object]) -> AliasedCostEntry: + return AliasedCostEntry( + {**entry, **values}, + entry.names, + registered_fields=entry.registered_fields, + created_by_registration=entry.created_by_registration, + registered_by=entry.registered_by, + written_fields=entry.written_fields, + displaced_values=entry.displaced_values, + saved_write_fields=entry.saved_write_fields, + latest_unsaved_write_fields=entry.latest_unsaved_write_fields, + written_values=entry.written_values, ) diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index b16cf4f4c7a..045353816e1 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -2744,6 +2744,70 @@ def test_deleting_a_flagged_deployment_keeps_what_a_deployment_whose_id_names_th assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer +_ID_ALIAS_BACKEND: Final = "hosted_vllm/withdraw-test-id-alias" + + +def _id_alias_deployment(model_id: str, **model_info: object) -> DeploymentTypedDict: + return _cache_point_deployment(model_id, model=_ID_ALIAS_BACKEND, **model_info) + + +def test_deleting_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None: + catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) + router: Final = Router( + model_list=[_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer})] + ) + assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is not catalog_answer + + assert router.delete_deployment(id=_CACHE_POINT_DEPLOYMENT_MODEL) is not None + + assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer + + +def test_editing_the_flag_off_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None: + catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) + router: Final = Router( + model_list=[_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer})] + ) + + assert router.upsert_deployment(Deployment(**_id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL))) is not None + + assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer + + +def test_replacing_the_model_list_without_a_deployment_whose_id_resolves_to_a_catalog_row_gives_that_row_back() -> None: + catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) + other: Final = _cache_point_deployment("other", model="hosted_vllm/withdraw-test-other") + router: Final = Router( + model_list=[ + other, + _id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, **{_CACHE_POINT_FLAG: not catalog_answer}), + ] + ) + + router.set_model_list([other]) + + assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer + + +def test_deleting_a_flagged_deployment_keeps_what_each_deployment_whose_id_resolves_to_the_row_wrote() -> None: + row: Final = _shared_cache_point_row() + catalog_answer: Final = bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) + router: Final = Router( + model_list=[ + _id_alias_deployment(_CACHE_POINT_DEPLOYMENT_MODEL, withdraw_test_first_note="kept"), + _id_alias_deployment(f"bedrock/converse/{_CACHE_POINT_MODEL}", withdraw_test_second_note="kept"), + _cache_point_deployment("flagged", **{_CACHE_POINT_FLAG: not catalog_answer}), + ] + ) + assert litellm.model_cost[row].get("withdraw_test_first_note") == "kept" + + assert router.delete_deployment(id="flagged") is not None + + assert litellm.model_cost[row].get("withdraw_test_first_note") == "kept" + assert litellm.model_cost[row].get("withdraw_test_second_note") == "kept" + assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer + + def test_deleting_a_deployment_keeps_a_limit_registered_on_a_row_the_rebuild_leaves_alone() -> None: shared_row: Final = "azure/gpt-4o" responses_model: Final = "azure/responses/gpt-4o" @@ -2831,6 +2895,47 @@ def test_deleting_a_flagged_deployment_keeps_a_price_an_sdk_user_set_on_the_cata assert bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) is catalog_answer +def test_deleting_deployments_keeps_a_limit_an_sdk_user_set_in_place_after_a_sibling_wrote_it() -> None: + row: Final = _shared_cache_point_row() + written_limit: Final = _unseen_max_output_tokens(row) + router: Final = Router( + model_list=[ + _cache_point_deployment("plain"), + _cache_point_deployment("limited", max_output_tokens=written_limit), + ] + ) + litellm.model_cost[row]["max_output_tokens"] = written_limit + 1 + flagged: Final = _cache_point_deployment( + "flagged", **{_CACHE_POINT_FLAG: not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)} + ) + assert router.upsert_deployment(Deployment(**flagged)) is not None + + assert router.delete_deployment(id="flagged") is not None + assert litellm.model_cost[row].get("max_output_tokens") == written_limit + 1 + assert router.delete_deployment(id="plain") is not None + assert litellm.model_cost[row].get("max_output_tokens") == written_limit + 1 + + +def test_deleting_a_flagged_deployment_gives_back_a_limit_a_sibling_wrote_over_an_earlier_in_place_edit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + row: Final = _shared_cache_point_row() + written_limit: Final = _unseen_max_output_tokens(row) + monkeypatch.setitem(litellm.model_cost[row], "max_output_tokens", written_limit + 1) + router: Final = Router( + model_list=[ + _cache_point_deployment("limited", max_output_tokens=written_limit), + _cache_point_deployment( + "flagged", **{_CACHE_POINT_FLAG: not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL)} + ), + ] + ) + + assert router.delete_deployment(id="flagged") is not None + + assert litellm.model_cost[row].get("max_output_tokens") == written_limit + + def test_deleting_a_flagged_deployment_keeps_the_limit_a_sibling_spelling_the_same_row_differently_sets() -> None: row: Final = _shared_cache_point_row() sibling_limit: Final = _unseen_max_output_tokens(row) @@ -2959,6 +3064,33 @@ def test_deleting_one_of_many_differently_priced_deployments_rewrites_the_shared assert litellm.model_cost["sibling-7"].get("input_cost_per_token") == pytest.approx(8e-7) +def test_deleting_one_of_many_deployments_alternating_two_flags_rewrites_the_shared_row_once_per_flag() -> None: + row: Final = _shared_cache_point_row() + flag: Final = not bedrock_model_accepts_cache_points(_CACHE_POINT_MODEL) + host: Final = _CostMapWriteRecordingHost() + router: Final = Router.from_config( + build_router_config( + router_arguments( + model_list=[ + *( + _cache_point_deployment(f"sibling-{n}", **{_CACHE_POINT_FLAG: (flag, not flag)[n % 2]}) + for n in range(51) + ), + _cache_point_deployment("doomed", max_output_tokens=_unseen_max_output_tokens(row)), + ] + ) + ), + host=host, + ) + host.cost_map_writes.reset_mock() + + assert router.delete_deployment(id="doomed") is not None + + assert host.cost_map_writes.call_count == 2 + assert litellm.model_cost[row].get(_CACHE_POINT_FLAG) is flag + assert litellm.model_cost[row].get("max_output_tokens") == _catalog_field(row, "max_output_tokens") + + @pytest.mark.parametrize( ("registered_before_the_router", "deleted", "saved_flag_wins"), [(True, "plain", False), (False, "plain", True), (True, "flagged", True)], @@ -3006,9 +3138,7 @@ def test_editing_a_priced_deployment_keeps_its_price_on_its_row_throughout_the_e host: Final = _IdRowPriceRecordingHost("edited") router: Final = Router.from_config( build_router_config( - router_arguments( - model_list=[_priced_cache_point_deployment("edited", 3e-6, **{_CACHE_POINT_FLAG: True})] - ) + router_arguments(model_list=[_priced_cache_point_deployment("edited", 3e-6, **{_CACHE_POINT_FLAG: True})]) ), host=host, )