diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index b927df0c438..a52c6d468ca 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -262,6 +262,13 @@ class LowestCostLoggingHandler(CustomLogger): if item_output_cost is None: item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0) + deployment_params = _deployment.get("litellm_params") + item_cache_read_cost = ( + deployment_params.get("cache_read_input_token_cost") + if deployment_params and deployment_params.get("cache_read_input_token_cost") is not None + else item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) + ) + # if litellm["model"] is not in model_cost map -> use item_cost = $10 item_cost = item_input_cost + item_output_cost @@ -294,12 +301,12 @@ class LowestCostLoggingHandler(CustomLogger): ): # if user passed in tpm / rpm in the model_list continue else: - potential_deployments.append((_deployment, item_cost)) + potential_deployments.append((_deployment, item_cost, item_cache_read_cost)) if len(potential_deployments) == 0: return None - potential_deployments = sorted(potential_deployments, key=lambda x: x[1]) + potential_deployments = sorted(potential_deployments, key=lambda x: (x[1], x[2])) selected_deployment: Final = potential_deployments[0][0] return selected_deployment diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py new file mode 100644 index 00000000000..c95b16daeba --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -0,0 +1,88 @@ +#### What this tests #### +# cost-based routing must break input+output price ties on cache-read price + +import pytest + +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler + + +@pytest.mark.parametrize("cheaper_cache_first", [True, False]) +@pytest.mark.asyncio +async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_cache_first): + """ + Regression test for https://github.com/BerriAI/litellm/issues/38064 + + Two deployments with identical input+output price must be separated by their + cache-read price, not by whichever one happens to be listed first. The pricier + deployment omits a cache-read price to exercise the input-cost fallback. + """ + cheaper = { + "model_name": "cache-tie-test", + "litellm_params": { + "model": "openai/tie-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-08, + }, + "model_info": {"id": "cheaper-cache"}, + } + pricier = { + "model_name": "cache-tie-test", + "litellm_params": { + "model": "openai/tie-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"id": "pricier-cache"}, + } + model_list = [cheaper, pricier] if cheaper_cache_first else [pricier, cheaper] + + logger = LowestCostLoggingHandler(router_cache=DualCache()) + + selected = await logger.async_get_available_deployments( + model_group="cache-tie-test", healthy_deployments=model_list + ) + + assert selected["model_info"]["id"] == "cheaper-cache" + + +@pytest.mark.parametrize("free_cache_first", [True, False]) +@pytest.mark.asyncio +async def test_cost_routing_honors_zero_deployment_cache_read_cost(free_cache_first): + """ + A deployment-level cache_read_input_token_cost of 0 is a price, not a missing value. + + Truthiness checks treat it as unset and fall through to the cost map / input-cost + fallback, which ranks a deployment whose cache reads are free as the priciest one + in a tie. Providers that do not charge for cache reads make this a real config. + """ + free = { + "model_name": "cache-zero-test", + "litellm_params": { + "model": "openai/zero-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 0.0, + }, + "model_info": {"id": "free-cache"}, + } + paid = { + "model_name": "cache-zero-test", + "litellm_params": { + "model": "openai/zero-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-08, + }, + "model_info": {"id": "paid-cache"}, + } + model_list = [free, paid] if free_cache_first else [paid, free] + + logger = LowestCostLoggingHandler(router_cache=DualCache()) + + selected = await logger.async_get_available_deployments( + model_group="cache-zero-test", healthy_deployments=model_list + ) + + assert selected["model_info"]["id"] == "free-cache"