scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 136

Merge 8beb5622c9 into e6c4580a31

Browse source
This commit is contained in:
Anuj ojha 2026-08-27 19:34:38 -05:00 • committed by GitHub
commit e7a3e95e78
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 73 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm/router_strategy/lowest_cost.py
View file

@ -246,14 +246,21 @@ class LowestCostLoggingHandler(CustomLogger):
)
item_litellm_model_name = _deployment.get("litellm_params", {}).get("model")
item_litellm_model_cost_map = litellm.model_cost.get(item_litellm_model_name, {})
if not item_litellm_model_cost_map and item_litellm_model_name:
try:
item_litellm_model_cost_map = dict(
litellm.get_model_info(model=item_litellm_model_name)
)
except Exception:
item_litellm_model_cost_map = {}
# check if user provided input_cost_per_token and output_cost_per_token in litellm_params
item_input_cost = None
item_output_cost = None
if _deployment.get("litellm_params", {}).get("input_cost_per_token", None):
if _deployment.get("litellm_params", {}).get("input_cost_per_token") is not None:
item_input_cost = _deployment.get("litellm_params", {}).get("input_cost_per_token")
if _deployment.get("litellm_params", {}).get("output_cost_per_token", None):
if _deployment.get("litellm_params", {}).get("output_cost_per_token") is not None:
item_output_cost = _deployment.get("litellm_params", {}).get("output_cost_per_token")
if item_input_cost is None:

64
tests/local_testing/test_lowest_cost_routing.py
View file

@ -46,6 +46,70 @@ async def test_get_available_deployments():
assert selected_model["model_info"]["id"] == "groq-llama"
@pytest.mark.asyncio
async def test_get_available_deployments_provider_prefixed_model_pricing():
test_cache = DualCache()
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "openai/gpt-4o-mini"},
"model_info": {"id": "prefixed-openai"},
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/some-custom-deployment",
"input_cost_per_token": 0.5,
"output_cost_per_token": 0.5,
},
"model_info": {"id": "custom-priced"},
},
]
lowest_cost_logger = LowestCostLoggingHandler(
router_cache=test_cache,
)
selected_model = await lowest_cost_logger.async_get_available_deployments(
model_group="gpt-3.5-turbo", healthy_deployments=model_list
)
assert selected_model["model_info"]["id"] == "prefixed-openai"
@pytest.mark.asyncio
async def test_get_available_deployments_zero_custom_price_is_honored():
test_cache = DualCache()
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "anthropic/claude-3-5-haiku-latest",
"input_cost_per_token": 0,
"output_cost_per_token": 0,
},
"model_info": {"id": "byok-zero-cost"},
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/some-custom-deployment",
"input_cost_per_token": 0.5,
"output_cost_per_token": 0.5,
},
"model_info": {"id": "custom-priced"},
},
]
lowest_cost_logger = LowestCostLoggingHandler(
router_cache=test_cache,
)
selected_model = await lowest_cost_logger.async_get_available_deployments(
model_group="gpt-3.5-turbo", healthy_deployments=model_list
)
assert selected_model["model_info"]["id"] == "byok-zero-cost"
@pytest.mark.asyncio
async def test_get_available_deployments_custom_price():
from litellm._logging import verbose_router_logger