From 3e6889ca0b9056ce757cc89b0ff9b3f6aa90abfc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:45:23 -0800 Subject: [PATCH] fix(router): replay off-peak deployments in order since they can copy a base model's mode --- litellm/router_python_core.py | 10 +++--- .../unit/test_router_model_cost_isolation.py | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 24b517bfc8e..4e50bab9596 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -1142,8 +1142,10 @@ class _SharedCostWrite: return self.model_id in writers or not writers.isdisjoint(self.backend_keys) @property - def sets_mode(self) -> bool: - return isinstance(self.model_info.get("mode"), str) + def may_write_mode(self) -> bool: + """Whether its write can carry ``mode``: set directly, or copied with the backend entry an off-peak schedule + inherits.""" + return isinstance(self.model_info.get("mode"), str) or bool(self.model_info.get("off_peak_pricing")) def replay_key(self, targets: frozenset[str]) -> tuple[str | None, str]: """Equal for two entries only when replaying both writes the ``targets`` rows exactly as replaying one does. @@ -12513,10 +12515,10 @@ class PythonCore(DeprecatedRouterMembers): ) -> tuple[int, ...]: """The position of the last of each set of ``landing`` writes that leave the ``targets`` rows as one of them does. Each field keeps the last value written to it, so only the last write of each set counts, unless a write - sets ``mode``: what a ``mode`` write does depends on the mode the row holds, so then only back-to-back runs + may carry ``mode``: what a ``mode`` write does depends on the mode the row holds, so then only back-to-back runs collapse.""" keyed: Final = tuple((position, write.replay_key(targets)) for position, write in landing) - if any(write.sets_mode for _position, write in landing): + if any(write.may_write_mode for _position, write in landing): return tuple(tuple(run)[-1][0] for _replay_key, run in groupby(keyed, key=itemgetter(1))) return tuple(sorted({replay_key: position for position, replay_key in keyed}.values())) diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 045353816e1..67acfc9f3dd 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -2995,6 +2995,39 @@ def test_deleting_a_deployment_keeps_the_mode_its_siblings_registered_in_model_l assert litellm.model_cost[backend_model].get("mode") == registered_mode +def _openai_catalog_model_with_mode(mode: str) -> str: + return next( + key + for key, entry in GetModelCostMap.loaded_model_cost_map().items() + if (entry.get("mode"), entry.get("litellm_provider"), "/" in key) == (mode, "openai", False) + ) + + +def test_deleting_a_deployment_keeps_the_mode_off_peak_siblings_inherit_in_model_list_order() -> None: + backend_model: Final = "hosted_vllm/withdraw-test-inherited-mode-model" + off_peak: Final = {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-7} + responses_model: Final = _openai_catalog_model_with_mode("responses") + router: Final = Router( + model_list=[ + _cache_point_deployment("doomed", model=backend_model, max_input_tokens=5), + *( + _cache_point_deployment(model_id, model=backend_model, base_model=base_model, off_peak_pricing=off_peak) + for model_id, base_model in ( + ("responses-first", responses_model), + ("embedding", _openai_catalog_model_with_mode("embedding")), + ("responses-last", responses_model), + ) + ), + ] + ) + registered_mode: Final = litellm.model_cost[backend_model].get("mode") + assert registered_mode is not None + + assert router.delete_deployment(id="doomed") is not None + + assert litellm.model_cost[backend_model].get("mode") == registered_mode + + class _CostMapWriteRecordingHost(PythonHost): __slots__ = ("cost_map_writes",)