diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e135503d11d..8a00ffa4d37 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -46,7 +46,7 @@ from litellm.llms.azure_ai.cost_calculator import ( cost_per_token as azure_ai_cost_per_token, ) from litellm.llms.azure_ai.cost_calculator import ( - is_router_fee_entry as azure_ai_is_router_fee_entry, + is_azure_model_router as azure_ai_is_model_router_name, ) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.llms.bedrock.cost_calculation import ( @@ -1665,7 +1665,7 @@ def completion_cost( ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) - if custom_llm_provider == "azure_ai" and not azure_ai_is_router_fee_entry(model): + if custom_llm_provider == "azure_ai" and not azure_ai_is_model_router_name(model): model_for_additional_costs = request_model_for_cost if completion_response is not None: hidden_params = getattr(completion_response, "_hidden_params", None) or {} diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index e57ba055587..5934525eca3 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -11,7 +11,7 @@ from litellm.types.utils import Usage from litellm.utils import get_model_info -def _is_azure_model_router(model: str) -> bool: +def is_azure_model_router(model: str) -> bool: """ Check if the model is Azure AI Foundry Model Router. @@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl Returns: float: The flat cost in USD, or 0.0 if not applicable """ - if not _is_azure_model_router(model): + if not is_azure_model_router(model): return 0.0 model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai") router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0) @@ -69,7 +69,7 @@ def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier ) except Exception as e: - if not _is_azure_model_router(model): + if not is_azure_model_router(model): raise verbose_logger.debug( "Azure AI Model Router: model '%s' not in cost map, only the routing fee applies. Error: %s", model, e @@ -77,6 +77,16 @@ def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> return 0.0, 0.0 +def _router_fee_name(model: str, request_model: str | None) -> str | None: + if is_router_fee_entry(model): + return None + if is_azure_model_router(model): + return model + if request_model is not None and is_azure_model_router(request_model): + return request_model + return None + + def cost_per_token( model: str, usage: Usage, @@ -85,13 +95,15 @@ def cost_per_token( service_tier: str | None = None, ) -> tuple[float, float]: """ - Price the response model's own tokens for Azure AI, plus the Model Router fee when the caller names the - router as the request model. + Price the response model's own tokens for Azure AI, plus the Model Router fee exactly once when either the + priced name or request_model is a Model Router name. - completion_cost never passes request_model: it charges the fee once through + A response priced as the router entry itself already carries the fee, so nothing is added on top of it. A + router deployment name that is missing from the cost map prices at the fee alone. + + completion_cost passes only the priced name: when that name is a routed model it adds the fee itself through AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost - breakdown. A response priced as the router entry itself already carries the fee, so request_model adds - nothing on top of it, and a router deployment name that is missing from the cost map prices at zero here. + breakdown, and when the name is router-shaped the fee is already in the prompt cost returned here. Args: model: str, the model name without provider prefix (from response) @@ -107,6 +119,7 @@ def cost_per_token( ValueError: If a model that is not a Model Router name is missing from the cost map """ prompt_cost, completion_cost = _response_model_cost(model=model, usage=usage, service_tier=service_tier) - if request_model is None or not _is_azure_model_router(request_model) or is_router_fee_entry(model): + fee_name: Final = _router_fee_name(model=model, request_model=request_model) + if fee_name is None: return prompt_cost, completion_cost - return prompt_cost + calculate_azure_model_router_flat_cost(request_model, usage.prompt_tokens), completion_cost + return prompt_cost + calculate_azure_model_router_flat_cost(fee_name, usage.prompt_tokens), completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 483ebd2431c..674a8b98304 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4060,9 +4060,13 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/model_router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -10754,6 +10758,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", @@ -10771,6 +10776,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 483ebd2431c..674a8b98304 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4060,9 +4060,13 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/model_router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -10754,6 +10758,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", @@ -10771,6 +10776,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 0deb79d14d3..7df14b91741 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -11,9 +11,9 @@ import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.cost_calculator import ( - _is_azure_model_router, calculate_azure_model_router_flat_cost, cost_per_token, + is_azure_model_router, ) from litellm.types.utils import Choices, Message, ModelResponse, Usage from litellm.utils import get_model_info @@ -54,7 +54,7 @@ class TestAzureModelRouterDetection: ) def test_is_azure_model_router(self, model: str, expected: bool): """Test Azure Model Router detection.""" - assert _is_azure_model_router(model) == expected + assert is_azure_model_router(model) == expected class TestAzureModelRouterPrefix: @@ -130,11 +130,21 @@ def _routed_model_cost() -> tuple[float, float]: @pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterFlatCost: - """cost_per_token prices the response model only; the router fee is the cost breakdown's own line item.""" + """cost_per_token charges the router fee once, for whichever router name the caller gives it.""" - def test_unmapped_router_deployment_name_prices_at_zero(self) -> None: + def test_unmapped_router_deployment_name_prices_the_fee(self) -> None: usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - assert cost_per_token(model="azure-model-router", usage=usage) == (0.0, 0.0) + prompt_cost, completion_cost_usd = cost_per_token(model="azure-model-router", usage=usage) + assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 + + def test_router_deployment_name_as_both_names_charges_the_fee_once(self) -> None: + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + prompt_cost, completion_cost_usd = cost_per_token( + model="model_router/my-deployment", usage=usage, request_model="azure_ai/model_router/my-deployment" + ) + assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) def test_router_entry_prices_its_own_fee(self, router_entry_name: str) -> None: @@ -209,7 +219,8 @@ class TestAzureModelRouterFlatCost: @pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterCostBreakdown: - """completion_cost charges the router fee exactly once, as the cost breakdown's additional cost line.""" + """completion_cost charges the router fee exactly once: as the breakdown's additional cost line when a routed + model is priced as itself, inside the input cost when the priced name is the router.""" def test_unmapped_router_deployment_name_costs_only_the_fee(self) -> None: cost = completion_cost( @@ -219,7 +230,7 @@ class TestAzureModelRouterCostBreakdown: ) assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) - def test_fee_is_the_breakdown_line_item_for_an_unmapped_router_name(self) -> None: + def test_unmapped_router_name_carries_the_fee_as_its_input_cost(self) -> None: logging_obj = _router_logging("azure-model-router") cost = completion_cost( completion_response=_azure_ai_response("azure-model-router"), @@ -229,10 +240,8 @@ class TestAzureModelRouterCostBreakdown: ) breakdown = logging_obj.cost_breakdown assert breakdown is not None - assert breakdown["input_cost"] == 0.0 - assert breakdown.get("additional_costs") == pytest.approx( - {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 - ) + assert breakdown["input_cost"] == pytest.approx(ROUTED_FEE, rel=1e-9) + assert "additional_costs" not in breakdown assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) def test_router_request_with_routed_response_charges_the_fee_once(self) -> None: diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index fab9be1b42c..19b082edd8a 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -5,131 +5,34 @@ from typing import Final import pytest from pydantic import TypeAdapter -from litellm import cost_per_token, get_model_info +from litellm import completion_cost, cost_per_token from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.types.utils import TranscriptionResponse REPO_ROOT: Final = Path(__file__).parents[4] +MAIN_COST_MAP: Final = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) -AZURE_OPENAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" -FOUNDRY_AOAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" -FOUNDRY_COHERE_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/" -FOUNDRY_GROK_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/" +AZURE_PRICING_PREFIX: Final = "https://azure.microsoft.com/en-us/pricing/details/" +A_MILLION: Final = 1_000_000 @dataclass(frozen=True, slots=True) class TokenPricedCatalogModel: catalog_name: str - mode: str - source: str - input_cost_per_token: float - output_cost_per_token: float - max_input_tokens: int - max_output_tokens: int - cache_read_input_token_cost: float | None - deprecation_date: str | None - supported_flags: tuple[str, ...] + dollars_per_million_input: float + dollars_per_million_output: float TOKEN_PRICED_MODELS: Final = ( - TokenPricedCatalogModel( - catalog_name="gpt-chat-latest", - mode="chat", - source=AZURE_OPENAI_PRICING, - input_cost_per_token=5e-06, - output_cost_per_token=3e-05, - max_input_tokens=272000, - max_output_tokens=128000, - cache_read_input_token_cost=5e-07, - deprecation_date="2026-12-02", - supported_flags=( - "supports_function_calling", - "supports_prompt_caching", - "supports_reasoning", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), - TokenPricedCatalogModel( - catalog_name="codex-mini", - mode="responses", - source=AZURE_OPENAI_PRICING, - input_cost_per_token=1.5e-06, - output_cost_per_token=6e-06, - max_input_tokens=200000, - max_output_tokens=100000, - cache_read_input_token_cost=3.75e-07, - deprecation_date="2026-11-15", - supported_flags=( - "supports_function_calling", - "supports_prompt_caching", - "supports_reasoning", - "supports_vision", - ), - ), - TokenPricedCatalogModel( - catalog_name="model-router", - mode="chat", - source=FOUNDRY_AOAI_PRICING, - input_cost_per_token=1.4e-07, - output_cost_per_token=0.0, - max_input_tokens=200000, - max_output_tokens=32768, - cache_read_input_token_cost=None, - deprecation_date="2027-05-20", - supported_flags=(), - ), - TokenPricedCatalogModel( - catalog_name="cohere-command-a", - mode="chat", - source=FOUNDRY_COHERE_PRICING, - input_cost_per_token=2.5e-06, - output_cost_per_token=1e-05, - max_input_tokens=131072, - max_output_tokens=8182, - cache_read_input_token_cost=None, - deprecation_date=None, - supported_flags=("supports_function_calling", "supports_tool_choice"), - ), - TokenPricedCatalogModel( - catalog_name="grok-4-20-reasoning", - mode="chat", - source=FOUNDRY_GROK_PRICING, - input_cost_per_token=1.25e-06, - output_cost_per_token=2.5e-06, - max_input_tokens=262000, - max_output_tokens=8192, - cache_read_input_token_cost=None, - deprecation_date="2027-04-06", - supported_flags=( - "supports_function_calling", - "supports_reasoning", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), - TokenPricedCatalogModel( - catalog_name="grok-4-20-non-reasoning", - mode="chat", - source=FOUNDRY_GROK_PRICING, - input_cost_per_token=1.25e-06, - output_cost_per_token=2.5e-06, - max_input_tokens=262000, - max_output_tokens=8192, - cache_read_input_token_cost=None, - deprecation_date="2027-04-06", - supported_flags=( - "supports_function_calling", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), + TokenPricedCatalogModel("gpt-chat-latest", 5.0, 30.0), + TokenPricedCatalogModel("codex-mini", 1.5, 6.0), + TokenPricedCatalogModel("model-router", 0.14, 0.0), + TokenPricedCatalogModel("cohere-command-a", 2.5, 10.0), + TokenPricedCatalogModel("grok-4-20-reasoning", 1.25, 2.5), + TokenPricedCatalogModel("grok-4-20-non-reasoning", 1.25, 2.5), ) +GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") CATALOG_NAMES: Final = tuple(spec.catalog_name for spec in TOKEN_PRICED_MODELS) + ("whisper",) @@ -137,60 +40,74 @@ def _cost_map_entry(path: Path, catalog_name: str) -> dict[str, object]: return COST_MAP_ADAPTER.validate_json(path.read_bytes())[f"azure_ai/{catalog_name}"] +@pytest.mark.parametrize("catalog_name", CATALOG_NAMES) +def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None: + routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{catalog_name}") + assert (routed_model, provider) == (catalog_name, "azure_ai") + + @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_name_is_priced_and_routed(spec: TokenPricedCatalogModel) -> None: - routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{spec.catalog_name}") - assert (routed_model, provider) == (spec.catalog_name, "azure_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["litellm_provider"] == "azure_ai" - assert info["mode"] == spec.mode - assert info["input_cost_per_token"] == spec.input_cost_per_token - assert info["output_cost_per_token"] == spec.output_cost_per_token - assert info["cache_read_input_token_cost"] == spec.cache_read_input_token_cost - assert info["max_input_tokens"] == spec.max_input_tokens - assert info["max_output_tokens"] == spec.max_output_tokens - assert info["max_tokens"] == spec.max_output_tokens - for flag in spec.supported_flags: - assert info[flag] is True, flag +def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: + prompt_cost, completion_cost_usd = cost_per_token( + model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION + ) + assert prompt_cost == pytest.approx(spec.dollars_per_million_input) + assert completion_cost_usd == pytest.approx(spec.dollars_per_million_output) @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "spec", - [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], - ids=lambda spec: spec.catalog_name, -) -def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: - prompt_cost, completion_cost = cost_per_token( - model=f"azure_ai/{spec.catalog_name}", prompt_tokens=1_000_000, completion_tokens=1_000_000 +@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) +def test_azure_ai_catalog_name_prices_the_same_in_any_casing(spec: TokenPricedCatalogModel) -> None: + lowercase_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + upper_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0) + assert upper_cost == lowercase_cost + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES) +def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None: + uncached_prompt_cost, _ = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + cached_prompt_cost, _ = cost_per_token( + model=f"azure_ai/{catalog_name}", + prompt_tokens=A_MILLION, + completion_tokens=0, + cache_read_input_tokens=A_MILLION, ) - assert prompt_cost == pytest.approx(spec.input_cost_per_token * 1_000_000) - assert completion_cost == pytest.approx(spec.output_cost_per_token * 1_000_000) + assert uncached_prompt_cost > 0 + assert cached_prompt_cost == pytest.approx(uncached_prompt_cost) @pytest.mark.usefixtures("local_model_cost_map") def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: - routed_model, provider, _, _ = get_llm_provider(model="azure_ai/whisper") - assert (routed_model, provider) == ("whisper", "azure_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["mode"] == "audio_transcription" - assert info["input_cost_per_second"] == 0.0001 - assert info["output_cost_per_second"] == 0.0001 + transcription: Final = TranscriptionResponse(text="hello") + transcription._hidden_params = { # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter + "custom_llm_provider": "azure_ai", + "model": "azure_ai/whisper", + "audio_transcription_duration": 3600, + } + cost = completion_cost( + completion_response=transcription, + model="azure_ai/whisper", + custom_llm_provider="azure_ai", + call_type="atranscription", + ) + assert cost == pytest.approx(0.36) @pytest.mark.parametrize("catalog_name", CATALOG_NAMES) def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None: - main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", catalog_name) - backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json", catalog_name) + main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name) + backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name) - assert str(main_entry["source"]).startswith("https://azure.microsoft.com/en-us/pricing/details/") + assert str(main_entry["source"]).startswith(AZURE_PRICING_PREFIX) assert backup_entry == main_entry -@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_entry_carries_its_retirement_date(spec: TokenPricedCatalogModel) -> None: - entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", spec.catalog_name) - assert entry.get("deprecation_date") == spec.deprecation_date +def test_azure_ai_model_router_spellings_share_one_entry() -> None: + underscore_entry = _cost_map_entry(MAIN_COST_MAP, "model_router") + hyphen_entry = _cost_map_entry(MAIN_COST_MAP, "model-router") + + assert {k: v for k, v in underscore_entry.items() if k != "comment"} == { + k: v for k, v in hyphen_entry.items() if k != "comment" + }