diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index b927df0c438..9e50122d69f 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -246,14 +246,21 @@ class LowestCostLoggingHandler(CustomLogger): ) item_litellm_model_name = _deployment.get("litellm_params", {}).get("model") item_litellm_model_cost_map = litellm.model_cost.get(item_litellm_model_name, {}) + if not item_litellm_model_cost_map and item_litellm_model_name: + try: + item_litellm_model_cost_map = dict( + litellm.get_model_info(model=item_litellm_model_name) + ) + except Exception: + item_litellm_model_cost_map = {} # check if user provided input_cost_per_token and output_cost_per_token in litellm_params item_input_cost = None item_output_cost = None - if _deployment.get("litellm_params", {}).get("input_cost_per_token", None): + if _deployment.get("litellm_params", {}).get("input_cost_per_token") is not None: item_input_cost = _deployment.get("litellm_params", {}).get("input_cost_per_token") - if _deployment.get("litellm_params", {}).get("output_cost_per_token", None): + if _deployment.get("litellm_params", {}).get("output_cost_per_token") is not None: item_output_cost = _deployment.get("litellm_params", {}).get("output_cost_per_token") if item_input_cost is None: diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py index 5bf3a3ee98b..9ec5d719cf1 100644 --- a/tests/local_testing/test_lowest_cost_routing.py +++ b/tests/local_testing/test_lowest_cost_routing.py @@ -46,6 +46,70 @@ async def test_get_available_deployments(): assert selected_model["model_info"]["id"] == "groq-llama" +@pytest.mark.asyncio +async def test_get_available_deployments_provider_prefixed_model_pricing(): + test_cache = DualCache() + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"id": "prefixed-openai"}, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/some-custom-deployment", + "input_cost_per_token": 0.5, + "output_cost_per_token": 0.5, + }, + "model_info": {"id": "custom-priced"}, + }, + ] + lowest_cost_logger = LowestCostLoggingHandler( + router_cache=test_cache, + ) + + selected_model = await lowest_cost_logger.async_get_available_deployments( + model_group="gpt-3.5-turbo", healthy_deployments=model_list + ) + + assert selected_model["model_info"]["id"] == "prefixed-openai" + + +@pytest.mark.asyncio +async def test_get_available_deployments_zero_custom_price_is_honored(): + test_cache = DualCache() + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "anthropic/claude-3-5-haiku-latest", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + }, + "model_info": {"id": "byok-zero-cost"}, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/some-custom-deployment", + "input_cost_per_token": 0.5, + "output_cost_per_token": 0.5, + }, + "model_info": {"id": "custom-priced"}, + }, + ] + lowest_cost_logger = LowestCostLoggingHandler( + router_cache=test_cache, + ) + + selected_model = await lowest_cost_logger.async_get_available_deployments( + model_group="gpt-3.5-turbo", healthy_deployments=model_list + ) + + assert selected_model["model_info"]["id"] == "byok-zero-cost" + + @pytest.mark.asyncio async def test_get_available_deployments_custom_price(): from litellm._logging import verbose_router_logger