From 432722944c5df079bf9ff800ae407cb7b1c667db Mon Sep 17 00:00:00 2001 From: Srivatsa03 Date: Mon, 24 Aug 2026 19:43:30 -0500 Subject: [PATCH 1/2] fix(router): break cost-based-routing ties on cache-read price cost-based-routing scored each deployment on input plus output price alone, so two deployments whose input and output prices are equal tied, and a stable sort then decided the winner by position in model_list. cache_read_input_token_cost was never read, which is the price that dominates cache-heavy traffic such as a long fixed system prompt or RAG over a stable prefix. The cost map already ships such a tie: fireworks_ai and tencent both list deepseek-v4-flash at the same input and output price with cache-read differing 10x, so the cheaper deployment won only when it happened to be listed first. Use the cache-read price as a tie-break, falling back to the input price when a model declares none so a missing entry is never ranked as though its cache reads were free. Deployments whose input plus output already differ are unaffected. Fixes #38064 --- litellm/router_strategy/lowest_cost.py | 7 ++- .../router_strategy/test_lowest_cost.py | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/router_strategy/test_lowest_cost.py diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index b927df0c438..ea5f6741ef6 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -265,6 +265,9 @@ class LowestCostLoggingHandler(CustomLogger): # if litellm["model"] is not in model_cost map -> use item_cost = $10 item_cost = item_input_cost + item_output_cost + # falls back to the input price rather than 0 so a model with no cache-read entry is + # never ranked as though its cache reads were free + item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) @@ -294,12 +297,12 @@ class LowestCostLoggingHandler(CustomLogger): ): # if user passed in tpm / rpm in the model_list continue else: - potential_deployments.append((_deployment, item_cost)) + potential_deployments.append((_deployment, item_cost, item_cache_read_cost)) if len(potential_deployments) == 0: return None - potential_deployments = sorted(potential_deployments, key=lambda x: x[1]) + potential_deployments = sorted(potential_deployments, key=lambda x: (x[1], x[2])) selected_deployment: Final = potential_deployments[0][0] return selected_deployment diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py new file mode 100644 index 00000000000..224c7a8f91a --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -0,0 +1,61 @@ +#### What this tests #### +# cost-based-routing scored deployments on input + output price alone, so two +# deployments whose input and output prices tie were ordered only by their position +# in model_list. cache_read_input_token_cost can differ 10x across such a tie, which +# is the price that dominates cache-heavy traffic. Issue #38064. + +import pytest + +import litellm +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler + +MODEL_GROUP = "test-tied-price-model" +CHEAP_CACHE = "cheap-cache-read" +PRICEY_CACHE = "pricey-cache-read" + + +def _deployments(order): + return [ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": f"openai/{name}"}, + "model_info": {"id": name}, + } + for name in order + ] + + +@pytest.fixture +def tied_prices(): + """Two models with identical input and output prices, cache-read differing 10x.""" + for name, cache_read in ((PRICEY_CACHE, 2.8e-08), (CHEAP_CACHE, 2.8e-09)): + litellm.register_model( + { + f"openai/{name}": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": cache_read, + } + } + ) + yield + for name in (PRICEY_CACHE, CHEAP_CACHE): + litellm.model_cost.pop(f"openai/{name}", None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "order", + [[PRICEY_CACHE, CHEAP_CACHE], [CHEAP_CACHE, PRICEY_CACHE]], + ids=["pricey-listed-first", "cheap-listed-first"], +) +async def test_tied_input_output_price_breaks_on_cache_read_not_list_order(order, tied_prices): + handler = LowestCostLoggingHandler(router_cache=DualCache()) + healthy = _deployments(order) + + selected = await handler.async_get_available_deployments(model_group=MODEL_GROUP, healthy_deployments=healthy) + + assert selected["model_info"]["id"] == CHEAP_CACHE From 784f7b92a52f9a83a3852ef597115876e028530f Mon Sep 17 00:00:00 2001 From: Srivatsa03 Date: Tue, 25 Aug 2026 17:58:16 -0500 Subject: [PATCH 2/2] fix(router): honour deployment cache-read overrides in the cost tie-break The tie-break read the cache-read price from the shared model cost map only, so two deployments of the same backend model that set different litellm_params.cache_read_input_token_cost still tied and fell back to list order. Read the deployment override first, the way input and output prices already are, then the cost map. Treat an explicitly null price as unset at both levels rather than letting it into the sort key, where comparing None against a float would raise TypeError. Reported by Greptile in review. --- litellm/router_strategy/lowest_cost.py | 11 +++- .../router_strategy/test_lowest_cost.py | 61 +++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index ea5f6741ef6..cf590bab624 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -265,9 +265,14 @@ class LowestCostLoggingHandler(CustomLogger): # if litellm["model"] is not in model_cost map -> use item_cost = $10 item_cost = item_input_cost + item_output_cost - # falls back to the input price rather than 0 so a model with no cache-read entry is - # never ranked as though its cache reads were free - item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) + # deployment override first, mirroring input/output above, then the cost map. An + # absent or explicitly null price falls back to the input price rather than 0, so a + # model without one is never ranked as though its cache reads were free + item_cache_read_cost = _deployment.get("litellm_params", {}).get("cache_read_input_token_cost") + if item_cache_read_cost is None: + item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost") + if item_cache_read_cost is None: + item_cache_read_cost = item_input_cost item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py index 224c7a8f91a..5f22ae340f8 100644 --- a/tests/test_litellm/router_strategy/test_lowest_cost.py +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -59,3 +59,64 @@ async def test_tied_input_output_price_breaks_on_cache_read_not_list_order(order selected = await handler.async_get_available_deployments(model_group=MODEL_GROUP, healthy_deployments=healthy) assert selected["model_info"]["id"] == CHEAP_CACHE + + +@pytest.fixture +def tied_prices_no_cache_entry(): + """Same tie, but the cost map carries an explicit null cache-read price.""" + for name in (PRICEY_CACHE, CHEAP_CACHE): + litellm.register_model( + { + f"openai/{name}": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + } + } + ) + litellm.model_cost[f"openai/{name}"]["cache_read_input_token_cost"] = None + yield + for name in (PRICEY_CACHE, CHEAP_CACHE): + litellm.model_cost.pop(f"openai/{name}", None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "order", + [[PRICEY_CACHE, CHEAP_CACHE], [CHEAP_CACHE, PRICEY_CACHE]], + ids=["pricey-listed-first", "cheap-listed-first"], +) +async def test_deployment_level_cache_read_override_breaks_the_tie(order, tied_prices): + """A per-deployment cache-read price must win over the shared model-map entry, the same way + input and output prices already honour litellm_params.""" + handler = LowestCostLoggingHandler(router_cache=DualCache()) + overrides = {PRICEY_CACHE: 9e-07, CHEAP_CACHE: 1e-09} + healthy = [ + { + "model_name": MODEL_GROUP, + "litellm_params": { + "model": f"openai/{PRICEY_CACHE}", + "cache_read_input_token_cost": overrides[name], + }, + "model_info": {"id": name}, + } + for name in order + ] + + selected = await handler.async_get_available_deployments(model_group=MODEL_GROUP, healthy_deployments=healthy) + + assert selected["model_info"]["id"] == CHEAP_CACHE + + +@pytest.mark.asyncio +async def test_null_cache_read_price_does_not_break_sorting(tied_prices_no_cache_entry): + """An explicitly null cache-read price means unset, not zero, so it must fall back to the + input price instead of putting None into the sort key and raising TypeError.""" + handler = LowestCostLoggingHandler(router_cache=DualCache()) + + selected = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, healthy_deployments=_deployments([PRICEY_CACHE, CHEAP_CACHE]) + ) + + assert selected["model_info"]["id"] in (PRICEY_CACHE, CHEAP_CACHE)