From 3b199cd3da3fc97a6a373a1247fb798fa2353ca1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:11:58 -0700 Subject: [PATCH 1/8] fix(azure_ai): price seven Foundry catalog names and charge the model router fee once Add cost map entries for azure_ai/gpt-chat-latest, codex-mini, whisper, model-router, cohere-command-a, grok-4-20-reasoning, and grok-4-20-non-reasoning, priced from the live Azure AI Foundry and Azure OpenAI pricing pages and the Azure Retail Prices API. Skip the model router flat fee when the response model is the router entry itself, since the generic cost already priced that fee. Before, azure_ai/model_router charged it twice. Resolves LIT-3157 --- litellm/llms/azure_ai/cost_calculator.py | 76 +++----- ...odel_prices_and_context_window_backup.json | 130 +++++++++++++ model_prices_and_context_window.json | 130 +++++++++++++ .../azure_ai/test_azure_ai_cost_calculator.py | 26 +++ ...azure_ai_foundry_catalog_model_metadata.py | 176 ++++++++++++++++++ 5 files changed, 492 insertions(+), 46 deletions(-) create mode 100644 tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 95f536296a2..141148f06e7 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -56,6 +56,27 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl return 0.0 +ROUTER_FEE_ENTRY_NAMES: Final = frozenset({"model-router", "model_router"}) + + +def _prices_router_fee_itself(model: str) -> bool: + return model.lower().rsplit("/", 1)[-1] in ROUTER_FEE_ENTRY_NAMES + + +def _base_cost_per_token(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float] | None: + try: + return generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier + ) + except Exception as e: + if not _is_azure_model_router(model): + raise + verbose_logger.debug( + "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e + ) + return None + + def cost_per_token( model: str, usage: Usage, @@ -66,9 +87,9 @@ def cost_per_token( """ Calculate the cost per token for Azure AI models. - For Azure AI Foundry Model Router: - - Adds a flat cost of $0.14 per million input tokens (from model_prices_and_context_window.json) - - Plus the cost of the actual model used (handled by generic_cost_per_token) + For Azure AI Foundry Model Router the routing fee (the azure_ai/model_router entry, $0.14 per + million input tokens) is added on top of the routed model's cost. When the response model is + the router entry itself, generic_cost_per_token has already charged that fee. Args: model: str, the model name without provider prefix (from response) @@ -83,49 +104,12 @@ def cost_per_token( ValueError: If the model is not found in the cost map and cost cannot be calculated (except for Model Router models where we return just the routing flat cost) """ - prompt_cost = 0.0 - completion_cost = 0.0 - - # Determine if this was a model router request - # Check both the response model and the request model is_router_request: Final = _is_azure_model_router(model) or ( request_model is not None and _is_azure_model_router(request_model) ) - - # Calculate base cost using generic cost calculator - # This may raise an exception if the model is not in the cost map - try: - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider="azure_ai", - service_tier=service_tier, - ) - except Exception as e: - # For Model Router, the model name (e.g., "azure-model-router") may not be in the cost map - # because it's a routing service, not an actual model. In this case, we continue - # to calculate just the routing flat cost. - if not _is_azure_model_router(model): - # Re-raise for non-router models - they should have pricing defined - raise - verbose_logger.debug( - "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e - ) - - # Add flat cost for Azure Model Router - # The flat cost is defined in model_prices_and_context_window.json for azure_ai/model_router - if is_router_request: - # Use the request model for flat cost calculation if available, otherwise use response model - router_model_for_calc: Final = request_model if request_model else model - router_flat_cost: Final = calculate_azure_model_router_flat_cost(router_model_for_calc, usage.prompt_tokens) - - if router_flat_cost > 0: - verbose_logger.debug( - f"Azure AI Model Router flat cost: ${router_flat_cost:.6f} " - f"({usage.prompt_tokens} tokens × ${router_flat_cost / usage.prompt_tokens:.9f}/token)" - ) - - # Add flat cost to prompt cost - prompt_cost += router_flat_cost - - return prompt_cost, completion_cost + base_cost: Final = _base_cost_per_token(model=model, usage=usage, service_tier=service_tier) + prompt_cost, completion_cost = base_cost if base_cost is not None else (0.0, 0.0) + if not is_router_request or (base_cost is not None and _prices_router_fee_itself(model)): + return prompt_cost, completion_cost + router_flat_cost: Final = calculate_azure_model_router_flat_cost(request_model or model, usage.prompt_tokens) + return prompt_cost + router_flat_cost, completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b1ffc1583e4..6649fa831d7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3581,6 +3581,82 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure_ai/codex-mini": { + "cache_read_input_token_cost": 3.75e-07, + "deprecation_date": "2026-11-15", + "input_cost_per_token": 1.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "responses", + "output_cost_per_token": 6e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure_ai/whisper": { + "deprecation_date": "2026-12-15", + "input_cost_per_second": 0.0001, + "litellm_provider": "azure_ai", + "mode": "audio_transcription", + "output_cost_per_second": 0.0001, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" + }, "azure_ai/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, @@ -3991,6 +4067,17 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, + "azure_ai/model-router": { + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 0, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", + "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" + }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, @@ -10302,6 +10389,18 @@ "/v1/ocr" ] }, + "azure_ai/cohere-command-a": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 131072, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", + "supports_function_calling": true, + "supports_tool_choice": true + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, @@ -10653,6 +10752,37 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-4-20-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_reasoning": true + }, + "azure_ai/grok-4-20-non-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b1ffc1583e4..6649fa831d7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3581,6 +3581,82 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure_ai/codex-mini": { + "cache_read_input_token_cost": 3.75e-07, + "deprecation_date": "2026-11-15", + "input_cost_per_token": 1.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "responses", + "output_cost_per_token": 6e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure_ai/whisper": { + "deprecation_date": "2026-12-15", + "input_cost_per_second": 0.0001, + "litellm_provider": "azure_ai", + "mode": "audio_transcription", + "output_cost_per_second": 0.0001, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" + }, "azure_ai/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, @@ -3991,6 +4067,17 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, + "azure_ai/model-router": { + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 0, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", + "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" + }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, @@ -10302,6 +10389,18 @@ "/v1/ocr" ] }, + "azure_ai/cohere-command-a": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 131072, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", + "supports_function_calling": true, + "supports_tool_choice": true + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, @@ -10653,6 +10752,37 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-4-20-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_reasoning": true + }, + "azure_ai/grok-4-20-non-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 9612d97d946..80cd99bd46b 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -528,3 +528,29 @@ def test_mai_thinking_1_model_info_and_cost(local_model_cost_map): assert model_info["supports_function_calling"] is True assert prompt_cost == pytest.approx(2.0) assert completion_cost == pytest.approx(8.0) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) +def test_router_entry_as_response_model_charges_the_fee_once(router_entry_name: str) -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + prompt_cost, completion_cost = cost_per_token(model=router_entry_name, usage=usage) + assert prompt_cost == pytest.approx(0.14, rel=1e-9) + assert completion_cost == 0.0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_unmapped_router_deployment_name_still_charges_the_fee() -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + prompt_cost, completion_cost = cost_per_token(model="azure-model-router", usage=usage) + assert prompt_cost == pytest.approx(0.14, rel=1e-9) + assert completion_cost == 0.0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_routed_model_response_adds_the_fee_on_top() -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + routed_prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage) + prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage, request_model="azure_ai/model-router") + assert routed_prompt_cost > 0 + assert prompt_cost == pytest.approx(routed_prompt_cost + 0.14, rel=1e-9) diff --git a/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py new file mode 100644 index 00000000000..9c5ca26a89c --- /dev/null +++ b/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py @@ -0,0 +1,176 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm import cost_per_token, get_model_info +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +REPO_ROOT: Final = Path(__file__).parents[2] +COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) +AZURE_OPENAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" +FOUNDRY_AOAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" +FOUNDRY_COHERE_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/" +FOUNDRY_GROK_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/" + + +@dataclass(frozen=True, slots=True) +class TokenPricedCatalogModel: + catalog_name: str + mode: str + source: str + input_cost_per_token: float + output_cost_per_token: float + max_input_tokens: int + max_output_tokens: int + cache_read_input_token_cost: float | None + supported_flags: tuple[str, ...] + + +TOKEN_PRICED_MODELS: Final = ( + TokenPricedCatalogModel( + catalog_name="gpt-chat-latest", + mode="chat", + source=AZURE_OPENAI_PRICING, + input_cost_per_token=5e-06, + output_cost_per_token=3e-05, + max_input_tokens=200000, + max_output_tokens=128000, + cache_read_input_token_cost=5e-07, + supported_flags=( + "supports_function_calling", + "supports_prompt_caching", + "supports_reasoning", + "supports_response_schema", + "supports_tool_choice", + "supports_vision", + "supports_web_search", + ), + ), + TokenPricedCatalogModel( + catalog_name="codex-mini", + mode="responses", + source=AZURE_OPENAI_PRICING, + input_cost_per_token=1.5e-06, + output_cost_per_token=6e-06, + max_input_tokens=200000, + max_output_tokens=100000, + cache_read_input_token_cost=3.75e-07, + supported_flags=("supports_function_calling", "supports_prompt_caching", "supports_reasoning", "supports_vision"), + ), + TokenPricedCatalogModel( + catalog_name="model-router", + mode="chat", + source=FOUNDRY_AOAI_PRICING, + input_cost_per_token=1.4e-07, + output_cost_per_token=0.0, + max_input_tokens=1048576, + max_output_tokens=32768, + cache_read_input_token_cost=None, + supported_flags=(), + ), + TokenPricedCatalogModel( + catalog_name="cohere-command-a", + mode="chat", + source=FOUNDRY_COHERE_PRICING, + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + max_input_tokens=131072, + max_output_tokens=4096, + cache_read_input_token_cost=None, + supported_flags=("supports_function_calling", "supports_tool_choice"), + ), + TokenPricedCatalogModel( + catalog_name="grok-4-20-reasoning", + mode="chat", + source=FOUNDRY_GROK_PRICING, + input_cost_per_token=1.25e-06, + output_cost_per_token=2.5e-06, + max_input_tokens=262000, + max_output_tokens=8192, + cache_read_input_token_cost=None, + supported_flags=( + "supports_function_calling", + "supports_reasoning", + "supports_response_schema", + "supports_tool_choice", + "supports_vision", + "supports_web_search", + ), + ), + TokenPricedCatalogModel( + catalog_name="grok-4-20-non-reasoning", + mode="chat", + source=FOUNDRY_GROK_PRICING, + input_cost_per_token=1.25e-06, + output_cost_per_token=2.5e-06, + max_input_tokens=262000, + max_output_tokens=8192, + cache_read_input_token_cost=None, + supported_flags=( + "supports_function_calling", + "supports_response_schema", + "supports_tool_choice", + "supports_vision", + "supports_web_search", + ), + ), +) +CATALOG_NAMES: Final = tuple(spec.catalog_name for spec in TOKEN_PRICED_MODELS) + ("whisper",) + + +def _cost_map_entry(path: Path, catalog_name: str) -> dict[str, object]: + return COST_MAP_ADAPTER.validate_json(path.read_bytes())[f"azure_ai/{catalog_name}"] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) +def test_azure_ai_catalog_name_is_priced_and_routed(spec: TokenPricedCatalogModel) -> None: + routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{spec.catalog_name}") + assert (routed_model, provider) == (spec.catalog_name, "azure_ai") + + info = get_model_info(model=routed_model, custom_llm_provider=provider) + assert info["litellm_provider"] == "azure_ai" + assert info["mode"] == spec.mode + assert info["input_cost_per_token"] == spec.input_cost_per_token + assert info["output_cost_per_token"] == spec.output_cost_per_token + assert info["cache_read_input_token_cost"] == spec.cache_read_input_token_cost + assert info["max_input_tokens"] == spec.max_input_tokens + assert info["max_output_tokens"] == spec.max_output_tokens + assert info["max_tokens"] == spec.max_output_tokens + for flag in spec.supported_flags: + assert info[flag] is True, flag + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "spec", [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], ids=lambda spec: spec.catalog_name +) +def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: + prompt_cost, completion_cost = cost_per_token( + model=f"azure_ai/{spec.catalog_name}", prompt_tokens=1_000_000, completion_tokens=1_000_000 + ) + assert prompt_cost == pytest.approx(spec.input_cost_per_token * 1_000_000) + assert completion_cost == pytest.approx(spec.output_cost_per_token * 1_000_000) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: + routed_model, provider, _, _ = get_llm_provider(model="azure_ai/whisper") + assert (routed_model, provider) == ("whisper", "azure_ai") + + info = get_model_info(model=routed_model, custom_llm_provider=provider) + assert info["mode"] == "audio_transcription" + assert info["input_cost_per_second"] == 0.0001 + assert info["output_cost_per_second"] == 0.0001 + + +@pytest.mark.parametrize("catalog_name", CATALOG_NAMES) +def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None: + main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", catalog_name) + backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json", catalog_name) + + assert str(main_entry["source"]).startswith("https://azure.microsoft.com/en-us/pricing/details/") + assert backup_entry == main_entry From 95402ccb711cdcd93f1c296579b18ec8bd32cab4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:36:40 -0700 Subject: [PATCH 2/8] test(azure_ai): move the Foundry catalog metadata test into the mapped azure_ai directory The new metadata test sat at the top of tests/test_litellm. The azure_ai metadata tests live in tests/test_litellm/llms/azure_ai next to the cost calculator test, so this moves it there and bumps its repo-root lookup by the two extra directory levels. No test changes. --- .../azure_ai}/test_azure_ai_foundry_catalog_model_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/test_litellm/{ => llms/azure_ai}/test_azure_ai_foundry_catalog_model_metadata.py (99%) diff --git a/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py similarity index 99% rename from tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py rename to tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 9c5ca26a89c..1b4e83438a6 100644 --- a/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -8,7 +8,7 @@ from pydantic import TypeAdapter from litellm import cost_per_token, get_model_info from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -REPO_ROOT: Final = Path(__file__).parents[2] +REPO_ROOT: Final = Path(__file__).parents[4] COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) AZURE_OPENAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" FOUNDRY_AOAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" From 415bdbfd8f6ba9bd0422087cc34509bde962ce55 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:46:39 -0700 Subject: [PATCH 3/8] fix(azure_ai): charge the Model Router fee once and correct catalog limits The router fee was folded into azure_ai.cost_per_token and then added again by the additional_costs hook, so every routed request paid it twice. The hook now owns the fee, the entry named by the deployment supplies the price, and a response priced as the router entry itself is not charged again model-router, gpt-chat-latest and cohere-command-a carry the limits from the Foundry models page, and model-router and grok-4-20-* carry their retirement dates. The router tests now run at the completion_cost level with a Logging object, which is the path the proxy takes, and fail at the merge base --- litellm/cost_calculator.py | 9 +- litellm/llms/azure_ai/cost_calculator.py | 87 ++- ...odel_prices_and_context_window_backup.json | 11 +- model_prices_and_context_window.json | 11 +- .../azure_ai/test_azure_ai_cost_calculator.py | 561 ++++++------------ ...azure_ai_foundry_catalog_model_metadata.py | 30 +- 6 files changed, 246 insertions(+), 463 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9a9d2ceda03..fc896a098b3 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -45,6 +45,9 @@ from litellm.llms.azure.cost_calculation import ( from litellm.llms.azure_ai.cost_calculator import ( cost_per_token as azure_ai_cost_per_token, ) +from litellm.llms.azure_ai.cost_calculator import ( + is_router_fee_entry as azure_ai_is_router_fee_entry, +) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.llms.bedrock.cost_calculation import ( cost_per_token as bedrock_cost_per_token, @@ -338,8 +341,6 @@ def cost_per_token( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") response: Any | None = None, - ### REQUEST MODEL ### - request_model: str | None = None, # original request model for router detection ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -661,7 +662,6 @@ def cost_per_token( model=model, usage=usage_block, response_time_ms=response_time_ms, - request_model=request_model, service_tier=service_tier, ) else: @@ -1659,11 +1659,10 @@ def completion_cost( data_residency=data_residency, vertex_location=vertex_location, response=completion_response, - request_model=request_model_for_cost, ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) - if custom_llm_provider == "azure_ai": + if custom_llm_provider == "azure_ai" and not azure_ai_is_router_fee_entry(model): model_for_additional_costs = request_model_for_cost if completion_response is not None: hidden_params = getattr(completion_response, "_hidden_params", None) or {} diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 141148f06e7..8a48860c0c4 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -31,6 +31,18 @@ def _is_azure_model_router(model: str) -> bool: return "model-router" in model_lower or "model_router" in model_lower or model_lower == "azure-model-router" +ROUTER_FEE_ENTRY_NAMES: Final = frozenset({"model-router", "model_router"}) + + +def is_router_fee_entry(model: str) -> bool: + return model.lower().removeprefix("azure_ai/") in ROUTER_FEE_ENTRY_NAMES + + +def _router_fee_entry_name(model: str) -> str: + entry_name: Final = model.lower().removeprefix("azure_ai/") + return entry_name if entry_name in ROUTER_FEE_ENTRY_NAMES else "model_router" + + def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> float: """ Calculate the flat cost for Azure AI Foundry Model Router. @@ -44,26 +56,39 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl """ if not _is_azure_model_router(model): return 0.0 - - # Get the model router pricing from model_prices_and_context_window.json - # Use "model_router" as the key (without actual model name suffix) - model_info: Final = get_model_info(model="model_router", custom_llm_provider="azure_ai") + model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai") router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0) - if router_flat_cost_per_token and router_flat_cost_per_token > 0: return prompt_tokens * router_flat_cost_per_token - return 0.0 -ROUTER_FEE_ENTRY_NAMES: Final = frozenset({"model-router", "model_router"}) +def cost_per_token( + model: str, + usage: Usage, + response_time_ms: float | None = 0.0, + service_tier: str | None = None, +) -> tuple[float, float]: + """ + Price the response model's own tokens for Azure AI. + The Azure AI Foundry Model Router fee is not part of this: completion_cost charges it once through + AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost + breakdown, and a response priced as the router entry itself already carries it. A router deployment name + that is missing from the cost map prices at zero here so that line item is the whole cost. -def _prices_router_fee_itself(model: str) -> bool: - return model.lower().rsplit("/", 1)[-1] in ROUTER_FEE_ENTRY_NAMES + Args: + model: str, the model name without provider prefix (from response) + usage: LiteLLM Usage block + response_time_ms: Optional response time in milliseconds + service_tier: Optional service tier the request was priced on + Returns: + Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd -def _base_cost_per_token(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float] | None: + Raises: + ValueError: If a model that is not a Model Router name is missing from the cost map + """ try: return generic_cost_per_token( model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier @@ -72,44 +97,6 @@ def _base_cost_per_token(model: str, usage: Usage, service_tier: str | None) -> if not _is_azure_model_router(model): raise verbose_logger.debug( - "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e + "Azure AI Model Router: model '%s' not in cost map, only the routing fee applies. Error: %s", model, e ) - return None - - -def cost_per_token( - model: str, - usage: Usage, - response_time_ms: float | None = 0.0, - request_model: str | None = None, - service_tier: str | None = None, -) -> tuple[float, float]: - """ - Calculate the cost per token for Azure AI models. - - For Azure AI Foundry Model Router the routing fee (the azure_ai/model_router entry, $0.14 per - million input tokens) is added on top of the routed model's cost. When the response model is - the router entry itself, generic_cost_per_token has already charged that fee. - - Args: - model: str, the model name without provider prefix (from response) - usage: LiteLLM Usage block - response_time_ms: Optional response time in milliseconds - request_model: Optional[str], the original request model name (to detect router usage) - - Returns: - Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - - Raises: - ValueError: If the model is not found in the cost map and cost cannot be calculated - (except for Model Router models where we return just the routing flat cost) - """ - is_router_request: Final = _is_azure_model_router(model) or ( - request_model is not None and _is_azure_model_router(request_model) - ) - base_cost: Final = _base_cost_per_token(model=model, usage=usage, service_tier=service_tier) - prompt_cost, completion_cost = base_cost if base_cost is not None else (0.0, 0.0) - if not is_router_request or (base_cost is not None and _prices_router_fee_itself(model)): - return prompt_cost, completion_cost - router_flat_cost: Final = calculate_azure_model_router_flat_cost(request_model or model, usage.prompt_tokens) - return prompt_cost + router_flat_cost, completion_cost + return 0.0, 0.0 diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6649fa831d7..483ebd2431c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3586,7 +3586,7 @@ "deprecation_date": "2026-12-02", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 200000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -4068,10 +4068,11 @@ "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure_ai/model-router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", - "max_input_tokens": 1048576, + "max_input_tokens": 200000, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -10393,8 +10394,8 @@ "input_cost_per_token": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8182, + "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", @@ -10753,6 +10754,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -10769,6 +10771,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6649fa831d7..483ebd2431c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3586,7 +3586,7 @@ "deprecation_date": "2026-12-02", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 200000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -4068,10 +4068,11 @@ "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure_ai/model-router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", - "max_input_tokens": 1048576, + "max_input_tokens": 200000, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -10393,8 +10394,8 @@ "input_cost_per_token": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8182, + "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", @@ -10753,6 +10754,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -10769,6 +10771,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 80cd99bd46b..20d0ec03a2a 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -2,20 +2,25 @@ Test Azure AI cost calculator, especially Model Router flat cost. """ +from datetime import datetime +from typing import Final + import pytest +import litellm +from litellm.cost_calculator import completion_cost +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.cost_calculator import ( _is_azure_model_router, + calculate_azure_model_router_flat_cost, cost_per_token, ) -from litellm.types.utils import Usage +from litellm.types.utils import Choices, Message, ModelResponse, Usage from litellm.utils import get_model_info # Get the flat cost from model_prices_and_context_window.json _model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai") -AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS = ( - _model_info.get("input_cost_per_token", 0) * 1_000_000 -) +AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS = _model_info.get("input_cost_per_token", 0) * 1_000_000 class TestAzureModelRouterDetection: @@ -80,377 +85,172 @@ class TestAzureModelRouterPrefix: assert result == expected +ROUTER_FEE_PER_TOKEN: Final = AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 +ROUTED_MODEL: Final = "gpt-4.1-nano-2025-04-14" +ROUTED_USAGE: Final = Usage(prompt_tokens=5000, completion_tokens=2000, total_tokens=7000) +ROUTED_FEE: Final = 5000 * ROUTER_FEE_PER_TOKEN + + +def _router_logging(request_model: str) -> Logging: + return Logging( + model=request_model, + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="test-123", + function_id="test-function", + ) + + +def _azure_ai_response(response_model: str, litellm_model_name: str | None = None) -> ModelResponse: + response: Final = ModelResponse( + id="test-123", + choices=[Choices(finish_reason="stop", index=0, message=Message(role="assistant", content="Hello"))], + created=1234567890, + model=response_model, + object="chat.completion", + usage=ROUTED_USAGE, + ) + response._hidden_params = ( + {"custom_llm_provider": "azure_ai"} + if litellm_model_name is None + else {"custom_llm_provider": "azure_ai", "litellm_model_name": litellm_model_name} + ) + return response + + +def _routed_model_cost() -> tuple[float, float]: + routed_info: Final = get_model_info(model=ROUTED_MODEL, custom_llm_provider="azure_ai") + return ( + ROUTED_USAGE.prompt_tokens * (routed_info["input_cost_per_token"] or 0.0), + ROUTED_USAGE.completion_tokens * (routed_info["output_cost_per_token"] or 0.0), + ) + + +@pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterFlatCost: - """Test Azure AI Foundry Model Router flat cost calculation.""" + """cost_per_token prices the response model only; the router fee is the cost breakdown's own line item.""" - def test_model_router_flat_cost_basic(self): - """Test that flat cost is added for Model Router requests.""" - model = "azure-model-router" - usage = Usage( - prompt_tokens=1000, - completion_tokens=500, - total_tokens=1500, + def test_unmapped_router_deployment_name_prices_at_zero(self) -> None: + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + assert cost_per_token(model="azure-model-router", usage=usage) == (0.0, 0.0) + + @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) + def test_router_entry_prices_its_own_fee(self, router_entry_name: str) -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + prompt_cost, completion_cost_usd = cost_per_token(model=router_entry_name, usage=usage) + assert prompt_cost == pytest.approx(0.14, rel=1e-9) + assert completion_cost_usd == 0.0 + + def test_routed_model_is_priced_as_itself(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + prompt_cost, completion_cost_usd = cost_per_token(model=ROUTED_MODEL, usage=ROUTED_USAGE) + assert routed_prompt_cost > 0 + assert prompt_cost == pytest.approx(routed_prompt_cost, rel=1e-9) + assert completion_cost_usd == pytest.approx(routed_completion_cost, rel=1e-9) + + def test_unmapped_model_that_is_not_a_router_name_raises(self) -> None: + usage = Usage(prompt_tokens=10, completion_tokens=10, total_tokens=20) + with pytest.raises(Exception, match="no-such-azure-ai-model"): + cost_per_token(model="no-such-azure-ai-model", usage=usage) + + def test_flat_cost_helper(self) -> None: + assert calculate_azure_model_router_flat_cost( + model="azure-model-router", prompt_tokens=10_000 + ) == pytest.approx(0.0014, rel=1e-9) + assert calculate_azure_model_router_flat_cost(model="gpt-5-nano", prompt_tokens=10_000) == 0.0 + + def test_flat_cost_reads_the_fee_from_the_deployment_named_entry(self) -> None: + litellm.register_model( + {"azure_ai/model-router": {"input_cost_per_token": 2e-07, "litellm_provider": "azure_ai", "mode": "chat"}} ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Calculate expected flat cost - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 + litellm.get_model_info.cache_clear() + assert calculate_azure_model_router_flat_cost(model="model-router", prompt_tokens=1_000_000) == pytest.approx( + 0.2, rel=1e-9 ) - - # Flat cost should be $0.00014 (1000 tokens × $0.14 / 1M tokens) - assert expected_flat_cost == pytest.approx(0.00014, rel=1e-9) - - # Prompt cost should include the flat cost - # (plus any base cost from the actual model used, which might be 0 if not in model_cost) - assert prompt_cost >= expected_flat_cost - print( - f"Model Router flat cost for {usage.prompt_tokens} tokens: ${expected_flat_cost:.6f}" - ) - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_model_router_flat_cost_large_request(self): - """Test flat cost calculation for larger requests.""" - model = "model-router" - usage = Usage( - prompt_tokens=100_000, - completion_tokens=50_000, - total_tokens=150_000, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Calculate expected flat cost - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 - ) - - # Flat cost should be $0.014 (100k tokens × $0.14 / 1M tokens) - assert expected_flat_cost == pytest.approx(0.014, rel=1e-9) - # Use approx for floating-point comparison - assert prompt_cost >= expected_flat_cost or prompt_cost == pytest.approx( - expected_flat_cost, rel=1e-9 - ) - print( - f"Model Router flat cost for {usage.prompt_tokens} tokens: ${expected_flat_cost:.6f}" - ) - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_model_router_flat_cost_1m_tokens(self): - """Test flat cost for exactly 1 million input tokens.""" - model = "azure-model-router" - usage = Usage( - prompt_tokens=1_000_000, - completion_tokens=100_000, - total_tokens=1_100_000, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Calculate expected flat cost - expected_flat_cost = AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - - # Flat cost should be exactly $0.14 for 1M tokens - assert expected_flat_cost == pytest.approx(0.14, rel=1e-9) - assert prompt_cost >= expected_flat_cost - print(f"Model Router flat cost for 1M tokens: ${expected_flat_cost:.6f}") - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_non_model_router_no_flat_cost(self): - """Test that non-Model Router models don't get the flat cost.""" - model = "gpt-4o" - usage = Usage( - prompt_tokens=1000, - completion_tokens=500, - total_tokens=1500, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # No flat cost should be added for non-Model Router models - # The cost might be 0 or based on the model's pricing - print(f"Non-Model Router prompt cost: ${prompt_cost:.6f}") - # We just ensure it doesn't crash and returns valid values - assert prompt_cost >= 0 - assert completion_cost >= 0 - - def test_model_router_with_cached_tokens(self): - """Test Model Router flat cost with cached tokens.""" - model = "azure-model-router" - usage = Usage( - prompt_tokens=2000, - completion_tokens=800, - total_tokens=2800, - cache_read_input_tokens=500, - cache_creation_input_tokens=200, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Flat cost is based on ALL prompt tokens (including cached) - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 - ) - - assert expected_flat_cost == pytest.approx(0.00028, rel=1e-9) - assert prompt_cost >= expected_flat_cost - print( - f"Model Router flat cost with caching for {usage.prompt_tokens} tokens: ${expected_flat_cost:.6f}" - ) - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_router_flat_cost_when_response_has_actual_model(self): - """ - Test that router flat cost is added when request was via router but response - contains the actual model (e.g., gpt-5-nano). - - This is the key fix: Azure returns the actual model in the response, but we - must still add the router flat cost because the request was made via model router. - """ - usage = Usage( - prompt_tokens=10000, - completion_tokens=5000, - total_tokens=15000, - ) - - # Response model is the actual model Azure used (not a router name) - response_model = "gpt-5-nano-2025-08-07" - # Request model is the router - user called azure_ai/model_router/model-router - request_model = "azure_ai/model_router/model-router" - - prompt_cost, completion_cost = cost_per_token( - model=response_model, - usage=usage, - request_model=request_model, - ) - - # Expected: model cost (from gpt-5-nano) + router flat cost - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 - ) - assert expected_flat_cost == pytest.approx(0.0014, rel=1e-9) - - # Total cost should be model cost + flat cost - total_cost = prompt_cost + completion_cost - assert total_cost >= expected_flat_cost - - # Prompt cost should include both model prompt cost and router flat cost - assert prompt_cost >= expected_flat_cost + assert calculate_azure_model_router_flat_cost( + model="azure-model-router", prompt_tokens=1_000_000 + ) == pytest.approx(0.14, rel=1e-9) +@pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterCostBreakdown: - """Test that Azure Model Router flat cost is tracked in cost breakdown.""" + """completion_cost charges the router fee exactly once, as the cost breakdown's additional cost line.""" - def test_flat_cost_calculation_helper(self): - """Test that flat cost can be calculated using the helper function.""" - from litellm.llms.azure_ai.cost_calculator import ( - calculate_azure_model_router_flat_cost, - ) - - model = "azure-model-router" - prompt_tokens = 10000 - - # Calculate flat cost using helper function - flat_cost = calculate_azure_model_router_flat_cost( - model=model, prompt_tokens=prompt_tokens - ) - - # Expected flat cost - expected_flat_cost = ( - prompt_tokens * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 - ) - - assert flat_cost > 0 - assert flat_cost == pytest.approx(expected_flat_cost, rel=1e-9) - print(f"Flat cost calculated: ${flat_cost:.6f}") - - def test_flat_cost_integration_with_completion_cost(self): - """Test that flat cost is properly integrated into completion_cost calculation.""" - import litellm - from litellm.cost_calculator import completion_cost - from litellm.types.utils import Choices, Message, ModelResponse, Usage - - # Create a mock response for azure_ai model router - response = ModelResponse( - id="test-123", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - role="assistant", - content="Test response", - ), - ) - ], - created=1234567890, - model="azure-model-router", - object="chat.completion", - usage=Usage( - prompt_tokens=5000, - completion_tokens=2000, - total_tokens=7000, - ), - ) - - # Set hidden params for provider - response._hidden_params = {"custom_llm_provider": "azure_ai"} - - # Calculate cost + def test_unmapped_router_deployment_name_costs_only_the_fee(self) -> None: cost = completion_cost( - completion_response=response, + completion_response=_azure_ai_response("azure-model-router"), model="azure-model-router", custom_llm_provider="azure_ai", ) + assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) - # Expected flat cost - expected_flat_cost = ( - 5000 * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 - ) - - # Cost should include the flat cost (use approx for floating-point comparison) - assert cost >= expected_flat_cost or cost == pytest.approx( - expected_flat_cost, rel=1e-9 - ) - print(f"Total cost with flat fee: ${cost:.6f}") - print(f"Expected minimum flat cost: ${expected_flat_cost:.6f}") - - def test_additional_costs_in_cost_breakdown(self): - """Test that Azure Model Router flat cost appears in additional_costs dict.""" - from datetime import datetime - - from litellm.cost_calculator import completion_cost - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import Choices, Message, ModelResponse, Usage - - # Create logging object with required parameters - logging_obj = Logging( - model="azure-model-router", - messages=[{"role": "user", "content": "Hello"}], - stream=False, - call_type="completion", - start_time=datetime.now(), - litellm_call_id="test-123", - function_id="test-function", - ) - - # Create a mock response for azure_ai model router - response = ModelResponse( - id="test-123", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - role="assistant", - content="Test response", - ), - ) - ], - created=1234567890, - model="azure-model-router", - object="chat.completion", - usage=Usage( - prompt_tokens=5000, - completion_tokens=2000, - total_tokens=7000, - ), - ) - - # Set hidden params for provider - response._hidden_params = {"custom_llm_provider": "azure_ai"} - - # Calculate cost with logging object + def test_fee_is_the_breakdown_line_item_for_an_unmapped_router_name(self) -> None: + logging_obj = _router_logging("azure-model-router") cost = completion_cost( - completion_response=response, + completion_response=_azure_ai_response("azure-model-router"), model="azure-model-router", custom_llm_provider="azure_ai", litellm_logging_obj=logging_obj, ) - - # Check that cost breakdown contains additional_costs - assert hasattr(logging_obj, "cost_breakdown") - assert logging_obj.cost_breakdown is not None - assert "additional_costs" in logging_obj.cost_breakdown - assert isinstance(logging_obj.cost_breakdown["additional_costs"], dict) - - # Check that the Azure Model Router flat cost is in additional_costs - additional_costs = logging_obj.cost_breakdown["additional_costs"] - assert "Azure Model Router Flat Cost" in additional_costs - - # Verify the flat cost value - expected_flat_cost = ( - 5000 * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert breakdown["input_cost"] == 0.0 + assert breakdown.get("additional_costs") == pytest.approx( + {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 ) - actual_flat_cost = additional_costs["Azure Model Router Flat Cost"] - assert actual_flat_cost == pytest.approx(expected_flat_cost, rel=1e-9) + assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) - print(f"Additional costs in breakdown: {additional_costs}") - print(f"Azure Model Router Flat Cost: ${actual_flat_cost:.6f}") - - def test_additional_costs_when_response_has_actual_model_via_hidden_params(self): - """additional_costs populated when response has actual model but request was via model router (hidden_params).""" - from datetime import datetime - - from litellm.cost_calculator import completion_cost - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import Choices, Message, ModelResponse, Usage - - logging_obj = Logging( - model="gpt-4.1-nano-2025-04-14", - messages=[{"role": "user", "content": "Hello"}], - stream=False, - call_type="completion", - start_time=datetime.now(), - litellm_call_id="test-123", - function_id="test-function", - ) - response = ModelResponse( - id="test-123", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message(role="assistant", content="Hello"), - ) - ], - created=1234567890, - model="gpt-4.1-nano-2025-04-14", - object="chat.completion", - usage=Usage(prompt_tokens=5000, completion_tokens=2000, total_tokens=7000), - ) - response._hidden_params = { - "custom_llm_provider": "azure_ai", - "litellm_model_name": "azure_ai/model-router", - } + def test_router_request_with_routed_response_charges_the_fee_once(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + logging_obj = _router_logging("model-router") cost = completion_cost( - completion_response=response, - model="gpt-4.1-nano-2025-04-14", + completion_response=_azure_ai_response(ROUTED_MODEL), + model=ROUTED_MODEL, custom_llm_provider="azure_ai", litellm_logging_obj=logging_obj, ) - expected_flat_cost = ( - 5000 * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert breakdown["input_cost"] == pytest.approx(routed_prompt_cost, rel=1e-9) + assert breakdown["output_cost"] == pytest.approx(routed_completion_cost, rel=1e-9) + assert breakdown.get("additional_costs") == pytest.approx( + {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 ) - assert cost >= expected_flat_cost - assert logging_obj.cost_breakdown is not None - assert "additional_costs" in logging_obj.cost_breakdown - assert ( - "Azure Model Router Flat Cost" - in logging_obj.cost_breakdown["additional_costs"] + assert cost == pytest.approx(routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9) + + def test_routed_response_named_by_hidden_params_charges_the_fee_once(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + logging_obj = _router_logging(ROUTED_MODEL) + cost = completion_cost( + completion_response=_azure_ai_response(ROUTED_MODEL, litellm_model_name="azure_ai/model-router"), + model=ROUTED_MODEL, + custom_llm_provider="azure_ai", + litellm_logging_obj=logging_obj, ) - assert logging_obj.cost_breakdown["additional_costs"][ - "Azure Model Router Flat Cost" - ] == pytest.approx(expected_flat_cost, rel=1e-9) + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert breakdown["input_cost"] == pytest.approx(routed_prompt_cost, rel=1e-9) + assert breakdown.get("additional_costs") == pytest.approx( + {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 + ) + assert cost == pytest.approx(routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9) + + @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) + def test_response_priced_as_the_router_entry_charges_the_fee_once(self, router_entry_name: str) -> None: + logging_obj = _router_logging(router_entry_name) + cost = completion_cost( + completion_response=_azure_ai_response(router_entry_name), + model=router_entry_name, + custom_llm_provider="azure_ai", + litellm_logging_obj=logging_obj, + ) + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert "additional_costs" not in breakdown + assert breakdown["input_cost"] == pytest.approx(ROUTED_FEE, rel=1e-9) + assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) class TestAzureAIServiceTierCostCalculation: @@ -459,26 +259,27 @@ class TestAzureAIServiceTierCostCalculation: @pytest.fixture(autouse=True) def register_test_model(self): import litellm - litellm.register_model(model_cost={ - "test-azure-ai-model": { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "input_cost_per_token_priority": 0.01, - "output_cost_per_token_priority": 0.02, - "input_cost_per_token_flex": 0.0005, - "output_cost_per_token_flex": 0.001, - "litellm_provider": "azure_ai", - "max_tokens": 8192, + + litellm.register_model( + model_cost={ + "test-azure-ai-model": { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "input_cost_per_token_priority": 0.01, + "output_cost_per_token_priority": 0.02, + "input_cost_per_token_flex": 0.0005, + "output_cost_per_token_flex": 0.001, + "litellm_provider": "azure_ai", + "max_tokens": 8192, + } } - }) + ) def test_service_tier_priority_higher_cost(self): """Priority tier should cost more than standard for azure_ai.""" usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - standard_prompt, standard_completion = cost_per_token( - model="test-azure-ai-model", usage=usage - ) + standard_prompt, standard_completion = cost_per_token(model="test-azure-ai-model", usage=usage) priority_prompt, priority_completion = cost_per_token( model="test-azure-ai-model", usage=usage, service_tier="priority" ) @@ -490,12 +291,8 @@ class TestAzureAIServiceTierCostCalculation: """Flex tier should cost less than standard for azure_ai.""" usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - standard_prompt, standard_completion = cost_per_token( - model="test-azure-ai-model", usage=usage - ) - flex_prompt, flex_completion = cost_per_token( - model="test-azure-ai-model", usage=usage, service_tier="flex" - ) + standard_prompt, standard_completion = cost_per_token(model="test-azure-ai-model", usage=usage) + flex_prompt, flex_completion = cost_per_token(model="test-azure-ai-model", usage=usage, service_tier="flex") assert flex_prompt < standard_prompt assert flex_completion < standard_completion @@ -528,29 +325,3 @@ def test_mai_thinking_1_model_info_and_cost(local_model_cost_map): assert model_info["supports_function_calling"] is True assert prompt_cost == pytest.approx(2.0) assert completion_cost == pytest.approx(8.0) - - -@pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) -def test_router_entry_as_response_model_charges_the_fee_once(router_entry_name: str) -> None: - usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) - prompt_cost, completion_cost = cost_per_token(model=router_entry_name, usage=usage) - assert prompt_cost == pytest.approx(0.14, rel=1e-9) - assert completion_cost == 0.0 - - -@pytest.mark.usefixtures("local_model_cost_map") -def test_unmapped_router_deployment_name_still_charges_the_fee() -> None: - usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) - prompt_cost, completion_cost = cost_per_token(model="azure-model-router", usage=usage) - assert prompt_cost == pytest.approx(0.14, rel=1e-9) - assert completion_cost == 0.0 - - -@pytest.mark.usefixtures("local_model_cost_map") -def test_routed_model_response_adds_the_fee_on_top() -> None: - usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) - routed_prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage) - prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage, request_model="azure_ai/model-router") - assert routed_prompt_cost > 0 - assert prompt_cost == pytest.approx(routed_prompt_cost + 0.14, rel=1e-9) diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 1b4e83438a6..fab9be1b42c 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -26,6 +26,7 @@ class TokenPricedCatalogModel: max_input_tokens: int max_output_tokens: int cache_read_input_token_cost: float | None + deprecation_date: str | None supported_flags: tuple[str, ...] @@ -36,9 +37,10 @@ TOKEN_PRICED_MODELS: Final = ( source=AZURE_OPENAI_PRICING, input_cost_per_token=5e-06, output_cost_per_token=3e-05, - max_input_tokens=200000, + max_input_tokens=272000, max_output_tokens=128000, cache_read_input_token_cost=5e-07, + deprecation_date="2026-12-02", supported_flags=( "supports_function_calling", "supports_prompt_caching", @@ -58,7 +60,13 @@ TOKEN_PRICED_MODELS: Final = ( max_input_tokens=200000, max_output_tokens=100000, cache_read_input_token_cost=3.75e-07, - supported_flags=("supports_function_calling", "supports_prompt_caching", "supports_reasoning", "supports_vision"), + deprecation_date="2026-11-15", + supported_flags=( + "supports_function_calling", + "supports_prompt_caching", + "supports_reasoning", + "supports_vision", + ), ), TokenPricedCatalogModel( catalog_name="model-router", @@ -66,9 +74,10 @@ TOKEN_PRICED_MODELS: Final = ( source=FOUNDRY_AOAI_PRICING, input_cost_per_token=1.4e-07, output_cost_per_token=0.0, - max_input_tokens=1048576, + max_input_tokens=200000, max_output_tokens=32768, cache_read_input_token_cost=None, + deprecation_date="2027-05-20", supported_flags=(), ), TokenPricedCatalogModel( @@ -78,8 +87,9 @@ TOKEN_PRICED_MODELS: Final = ( input_cost_per_token=2.5e-06, output_cost_per_token=1e-05, max_input_tokens=131072, - max_output_tokens=4096, + max_output_tokens=8182, cache_read_input_token_cost=None, + deprecation_date=None, supported_flags=("supports_function_calling", "supports_tool_choice"), ), TokenPricedCatalogModel( @@ -91,6 +101,7 @@ TOKEN_PRICED_MODELS: Final = ( max_input_tokens=262000, max_output_tokens=8192, cache_read_input_token_cost=None, + deprecation_date="2027-04-06", supported_flags=( "supports_function_calling", "supports_reasoning", @@ -109,6 +120,7 @@ TOKEN_PRICED_MODELS: Final = ( max_input_tokens=262000, max_output_tokens=8192, cache_read_input_token_cost=None, + deprecation_date="2027-04-06", supported_flags=( "supports_function_calling", "supports_response_schema", @@ -146,7 +158,9 @@ def test_azure_ai_catalog_name_is_priced_and_routed(spec: TokenPricedCatalogMode @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - "spec", [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], ids=lambda spec: spec.catalog_name + "spec", + [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], + ids=lambda spec: spec.catalog_name, ) def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: prompt_cost, completion_cost = cost_per_token( @@ -174,3 +188,9 @@ def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> No assert str(main_entry["source"]).startswith("https://azure.microsoft.com/en-us/pricing/details/") assert backup_entry == main_entry + + +@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) +def test_azure_ai_catalog_entry_carries_its_retirement_date(spec: TokenPricedCatalogModel) -> None: + entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", spec.catalog_name) + assert entry.get("deprecation_date") == spec.deprecation_date From c02f2dc0feff1f95d61b1be699565a835402a122 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:10:54 -0700 Subject: [PATCH 4/8] fix(azure_ai): keep the request_model keyword on cost_per_token Restores the public keyword removed at 415bdbfd8f. A direct caller that names the Model Router as the request model gets the routing fee folded into the prompt cost once; completion_cost never passes it and charges the fee through the additional-costs hook as before --- litellm/cost_calculator.py | 3 + litellm/llms/azure_ai/cost_calculator.py | 62 +++++++++++-------- .../azure_ai/test_azure_ai_cost_calculator.py | 33 ++++++++++ 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index fc896a098b3..e135503d11d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -341,6 +341,8 @@ def cost_per_token( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") response: Any | None = None, + ### REQUEST MODEL ### + request_model: str | None = None, # original request model for router detection ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -662,6 +664,7 @@ def cost_per_token( model=model, usage=usage_block, response_time_ms=response_time_ms, + request_model=request_model, service_tier=service_tier, ) else: diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 8a48860c0c4..e57ba055587 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -63,32 +63,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl return 0.0 -def cost_per_token( - model: str, - usage: Usage, - response_time_ms: float | None = 0.0, - service_tier: str | None = None, -) -> tuple[float, float]: - """ - Price the response model's own tokens for Azure AI. - - The Azure AI Foundry Model Router fee is not part of this: completion_cost charges it once through - AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost - breakdown, and a response priced as the router entry itself already carries it. A router deployment name - that is missing from the cost map prices at zero here so that line item is the whole cost. - - Args: - model: str, the model name without provider prefix (from response) - usage: LiteLLM Usage block - response_time_ms: Optional response time in milliseconds - service_tier: Optional service tier the request was priced on - - Returns: - Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - - Raises: - ValueError: If a model that is not a Model Router name is missing from the cost map - """ +def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]: try: return generic_cost_per_token( model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier @@ -100,3 +75,38 @@ def cost_per_token( "Azure AI Model Router: model '%s' not in cost map, only the routing fee applies. Error: %s", model, e ) return 0.0, 0.0 + + +def cost_per_token( + model: str, + usage: Usage, + response_time_ms: float | None = 0.0, + request_model: str | None = None, + service_tier: str | None = None, +) -> tuple[float, float]: + """ + Price the response model's own tokens for Azure AI, plus the Model Router fee when the caller names the + router as the request model. + + completion_cost never passes request_model: it charges the fee once through + AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost + breakdown. A response priced as the router entry itself already carries the fee, so request_model adds + nothing on top of it, and a router deployment name that is missing from the cost map prices at zero here. + + Args: + model: str, the model name without provider prefix (from response) + usage: LiteLLM Usage block + response_time_ms: Optional response time in milliseconds + request_model: Optional[str], the original request model name; a Model Router name adds the routing fee + service_tier: Optional service tier the request was priced on + + Returns: + Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd + + Raises: + ValueError: If a model that is not a Model Router name is missing from the cost map + """ + prompt_cost, completion_cost = _response_model_cost(model=model, usage=usage, service_tier=service_tier) + if request_model is None or not _is_azure_model_router(request_model) or is_router_fee_entry(model): + return prompt_cost, completion_cost + return prompt_cost + calculate_azure_model_router_flat_cost(request_model, usage.prompt_tokens), completion_cost diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 20d0ec03a2a..0deb79d14d3 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -155,6 +155,39 @@ class TestAzureModelRouterFlatCost: with pytest.raises(Exception, match="no-such-azure-ai-model"): cost_per_token(model="no-such-azure-ai-model", usage=usage) + def test_request_model_through_the_router_adds_the_fee_once(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + prompt_cost, completion_cost_usd = cost_per_token( + model=ROUTED_MODEL, usage=ROUTED_USAGE, request_model="azure_ai/model-router" + ) + assert prompt_cost == pytest.approx(routed_prompt_cost + ROUTED_FEE, rel=1e-9) + assert completion_cost_usd == pytest.approx(routed_completion_cost, rel=1e-9) + + def test_request_model_that_is_not_the_router_adds_nothing(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + assert cost_per_token( + model=ROUTED_MODEL, usage=ROUTED_USAGE, request_model=f"azure_ai/{ROUTED_MODEL}" + ) == pytest.approx((routed_prompt_cost, routed_completion_cost), rel=1e-9) + + @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) + def test_request_model_does_not_double_the_router_entry(self, router_entry_name: str) -> None: + prompt_cost, completion_cost_usd = cost_per_token( + model=router_entry_name, usage=ROUTED_USAGE, request_model=f"azure_ai/{router_entry_name}" + ) + assert prompt_cost == pytest.approx(ROUTED_FEE, rel=1e-9) + assert completion_cost_usd == 0.0 + + def test_public_cost_per_token_keeps_the_request_model_keyword(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + prompt_cost, completion_cost_usd = litellm.cost_per_token( + model=ROUTED_MODEL, + custom_llm_provider="azure_ai", + usage_object=ROUTED_USAGE, + request_model="azure_ai/model-router", + ) + assert prompt_cost == pytest.approx(routed_prompt_cost + ROUTED_FEE, rel=1e-9) + assert completion_cost_usd == pytest.approx(routed_completion_cost, rel=1e-9) + def test_flat_cost_helper(self) -> None: assert calculate_azure_model_router_flat_cost( model="azure-model-router", prompt_tokens=10_000 From 55c10c1983c92dbad1d62dfc3c14aab94696639a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:34:34 -0700 Subject: [PATCH 5/8] fix(azure_ai): charge the router fee once for any router name and price grok-4-20 cache reads Direct litellm.cost_per_token callers that name a Model Router deployment as the model get the routing fee again, as they did before this branch, and the fee is still charged exactly once on every completion_cost path. The grok-4-20 entries bill cached prompt tokens at the input rate, since Azure has no cached-input meter for them, and the model_router twin carries the same limits and retirement date as model-router. The catalog test now exercises the cost calculator and map relations instead of pinning map fields. --- litellm/cost_calculator.py | 4 +- litellm/llms/azure_ai/cost_calculator.py | 33 ++- ...odel_prices_and_context_window_backup.json | 6 + model_prices_and_context_window.json | 6 + .../azure_ai/test_azure_ai_cost_calculator.py | 31 ++- ...azure_ai_foundry_catalog_model_metadata.py | 219 ++++++------------ 6 files changed, 125 insertions(+), 174 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e135503d11d..8a00ffa4d37 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -46,7 +46,7 @@ from litellm.llms.azure_ai.cost_calculator import ( cost_per_token as azure_ai_cost_per_token, ) from litellm.llms.azure_ai.cost_calculator import ( - is_router_fee_entry as azure_ai_is_router_fee_entry, + is_azure_model_router as azure_ai_is_model_router_name, ) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.llms.bedrock.cost_calculation import ( @@ -1665,7 +1665,7 @@ def completion_cost( ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) - if custom_llm_provider == "azure_ai" and not azure_ai_is_router_fee_entry(model): + if custom_llm_provider == "azure_ai" and not azure_ai_is_model_router_name(model): model_for_additional_costs = request_model_for_cost if completion_response is not None: hidden_params = getattr(completion_response, "_hidden_params", None) or {} diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index e57ba055587..5934525eca3 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -11,7 +11,7 @@ from litellm.types.utils import Usage from litellm.utils import get_model_info -def _is_azure_model_router(model: str) -> bool: +def is_azure_model_router(model: str) -> bool: """ Check if the model is Azure AI Foundry Model Router. @@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl Returns: float: The flat cost in USD, or 0.0 if not applicable """ - if not _is_azure_model_router(model): + if not is_azure_model_router(model): return 0.0 model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai") router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0) @@ -69,7 +69,7 @@ def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier ) except Exception as e: - if not _is_azure_model_router(model): + if not is_azure_model_router(model): raise verbose_logger.debug( "Azure AI Model Router: model '%s' not in cost map, only the routing fee applies. Error: %s", model, e @@ -77,6 +77,16 @@ def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> return 0.0, 0.0 +def _router_fee_name(model: str, request_model: str | None) -> str | None: + if is_router_fee_entry(model): + return None + if is_azure_model_router(model): + return model + if request_model is not None and is_azure_model_router(request_model): + return request_model + return None + + def cost_per_token( model: str, usage: Usage, @@ -85,13 +95,15 @@ def cost_per_token( service_tier: str | None = None, ) -> tuple[float, float]: """ - Price the response model's own tokens for Azure AI, plus the Model Router fee when the caller names the - router as the request model. + Price the response model's own tokens for Azure AI, plus the Model Router fee exactly once when either the + priced name or request_model is a Model Router name. - completion_cost never passes request_model: it charges the fee once through + A response priced as the router entry itself already carries the fee, so nothing is added on top of it. A + router deployment name that is missing from the cost map prices at the fee alone. + + completion_cost passes only the priced name: when that name is a routed model it adds the fee itself through AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost - breakdown. A response priced as the router entry itself already carries the fee, so request_model adds - nothing on top of it, and a router deployment name that is missing from the cost map prices at zero here. + breakdown, and when the name is router-shaped the fee is already in the prompt cost returned here. Args: model: str, the model name without provider prefix (from response) @@ -107,6 +119,7 @@ def cost_per_token( ValueError: If a model that is not a Model Router name is missing from the cost map """ prompt_cost, completion_cost = _response_model_cost(model=model, usage=usage, service_tier=service_tier) - if request_model is None or not _is_azure_model_router(request_model) or is_router_fee_entry(model): + fee_name: Final = _router_fee_name(model=model, request_model=request_model) + if fee_name is None: return prompt_cost, completion_cost - return prompt_cost + calculate_azure_model_router_flat_cost(request_model, usage.prompt_tokens), completion_cost + return prompt_cost + calculate_azure_model_router_flat_cost(fee_name, usage.prompt_tokens), completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 483ebd2431c..674a8b98304 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4060,9 +4060,13 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/model_router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -10754,6 +10758,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", @@ -10771,6 +10776,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 483ebd2431c..674a8b98304 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4060,9 +4060,13 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/model_router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -10754,6 +10758,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", @@ -10771,6 +10776,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 0deb79d14d3..7df14b91741 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -11,9 +11,9 @@ import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.cost_calculator import ( - _is_azure_model_router, calculate_azure_model_router_flat_cost, cost_per_token, + is_azure_model_router, ) from litellm.types.utils import Choices, Message, ModelResponse, Usage from litellm.utils import get_model_info @@ -54,7 +54,7 @@ class TestAzureModelRouterDetection: ) def test_is_azure_model_router(self, model: str, expected: bool): """Test Azure Model Router detection.""" - assert _is_azure_model_router(model) == expected + assert is_azure_model_router(model) == expected class TestAzureModelRouterPrefix: @@ -130,11 +130,21 @@ def _routed_model_cost() -> tuple[float, float]: @pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterFlatCost: - """cost_per_token prices the response model only; the router fee is the cost breakdown's own line item.""" + """cost_per_token charges the router fee once, for whichever router name the caller gives it.""" - def test_unmapped_router_deployment_name_prices_at_zero(self) -> None: + def test_unmapped_router_deployment_name_prices_the_fee(self) -> None: usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - assert cost_per_token(model="azure-model-router", usage=usage) == (0.0, 0.0) + prompt_cost, completion_cost_usd = cost_per_token(model="azure-model-router", usage=usage) + assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 + + def test_router_deployment_name_as_both_names_charges_the_fee_once(self) -> None: + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + prompt_cost, completion_cost_usd = cost_per_token( + model="model_router/my-deployment", usage=usage, request_model="azure_ai/model_router/my-deployment" + ) + assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) def test_router_entry_prices_its_own_fee(self, router_entry_name: str) -> None: @@ -209,7 +219,8 @@ class TestAzureModelRouterFlatCost: @pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterCostBreakdown: - """completion_cost charges the router fee exactly once, as the cost breakdown's additional cost line.""" + """completion_cost charges the router fee exactly once: as the breakdown's additional cost line when a routed + model is priced as itself, inside the input cost when the priced name is the router.""" def test_unmapped_router_deployment_name_costs_only_the_fee(self) -> None: cost = completion_cost( @@ -219,7 +230,7 @@ class TestAzureModelRouterCostBreakdown: ) assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) - def test_fee_is_the_breakdown_line_item_for_an_unmapped_router_name(self) -> None: + def test_unmapped_router_name_carries_the_fee_as_its_input_cost(self) -> None: logging_obj = _router_logging("azure-model-router") cost = completion_cost( completion_response=_azure_ai_response("azure-model-router"), @@ -229,10 +240,8 @@ class TestAzureModelRouterCostBreakdown: ) breakdown = logging_obj.cost_breakdown assert breakdown is not None - assert breakdown["input_cost"] == 0.0 - assert breakdown.get("additional_costs") == pytest.approx( - {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 - ) + assert breakdown["input_cost"] == pytest.approx(ROUTED_FEE, rel=1e-9) + assert "additional_costs" not in breakdown assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) def test_router_request_with_routed_response_charges_the_fee_once(self) -> None: diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index fab9be1b42c..19b082edd8a 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -5,131 +5,34 @@ from typing import Final import pytest from pydantic import TypeAdapter -from litellm import cost_per_token, get_model_info +from litellm import completion_cost, cost_per_token from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.types.utils import TranscriptionResponse REPO_ROOT: Final = Path(__file__).parents[4] +MAIN_COST_MAP: Final = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) -AZURE_OPENAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" -FOUNDRY_AOAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" -FOUNDRY_COHERE_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/" -FOUNDRY_GROK_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/" +AZURE_PRICING_PREFIX: Final = "https://azure.microsoft.com/en-us/pricing/details/" +A_MILLION: Final = 1_000_000 @dataclass(frozen=True, slots=True) class TokenPricedCatalogModel: catalog_name: str - mode: str - source: str - input_cost_per_token: float - output_cost_per_token: float - max_input_tokens: int - max_output_tokens: int - cache_read_input_token_cost: float | None - deprecation_date: str | None - supported_flags: tuple[str, ...] + dollars_per_million_input: float + dollars_per_million_output: float TOKEN_PRICED_MODELS: Final = ( - TokenPricedCatalogModel( - catalog_name="gpt-chat-latest", - mode="chat", - source=AZURE_OPENAI_PRICING, - input_cost_per_token=5e-06, - output_cost_per_token=3e-05, - max_input_tokens=272000, - max_output_tokens=128000, - cache_read_input_token_cost=5e-07, - deprecation_date="2026-12-02", - supported_flags=( - "supports_function_calling", - "supports_prompt_caching", - "supports_reasoning", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), - TokenPricedCatalogModel( - catalog_name="codex-mini", - mode="responses", - source=AZURE_OPENAI_PRICING, - input_cost_per_token=1.5e-06, - output_cost_per_token=6e-06, - max_input_tokens=200000, - max_output_tokens=100000, - cache_read_input_token_cost=3.75e-07, - deprecation_date="2026-11-15", - supported_flags=( - "supports_function_calling", - "supports_prompt_caching", - "supports_reasoning", - "supports_vision", - ), - ), - TokenPricedCatalogModel( - catalog_name="model-router", - mode="chat", - source=FOUNDRY_AOAI_PRICING, - input_cost_per_token=1.4e-07, - output_cost_per_token=0.0, - max_input_tokens=200000, - max_output_tokens=32768, - cache_read_input_token_cost=None, - deprecation_date="2027-05-20", - supported_flags=(), - ), - TokenPricedCatalogModel( - catalog_name="cohere-command-a", - mode="chat", - source=FOUNDRY_COHERE_PRICING, - input_cost_per_token=2.5e-06, - output_cost_per_token=1e-05, - max_input_tokens=131072, - max_output_tokens=8182, - cache_read_input_token_cost=None, - deprecation_date=None, - supported_flags=("supports_function_calling", "supports_tool_choice"), - ), - TokenPricedCatalogModel( - catalog_name="grok-4-20-reasoning", - mode="chat", - source=FOUNDRY_GROK_PRICING, - input_cost_per_token=1.25e-06, - output_cost_per_token=2.5e-06, - max_input_tokens=262000, - max_output_tokens=8192, - cache_read_input_token_cost=None, - deprecation_date="2027-04-06", - supported_flags=( - "supports_function_calling", - "supports_reasoning", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), - TokenPricedCatalogModel( - catalog_name="grok-4-20-non-reasoning", - mode="chat", - source=FOUNDRY_GROK_PRICING, - input_cost_per_token=1.25e-06, - output_cost_per_token=2.5e-06, - max_input_tokens=262000, - max_output_tokens=8192, - cache_read_input_token_cost=None, - deprecation_date="2027-04-06", - supported_flags=( - "supports_function_calling", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), + TokenPricedCatalogModel("gpt-chat-latest", 5.0, 30.0), + TokenPricedCatalogModel("codex-mini", 1.5, 6.0), + TokenPricedCatalogModel("model-router", 0.14, 0.0), + TokenPricedCatalogModel("cohere-command-a", 2.5, 10.0), + TokenPricedCatalogModel("grok-4-20-reasoning", 1.25, 2.5), + TokenPricedCatalogModel("grok-4-20-non-reasoning", 1.25, 2.5), ) +GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") CATALOG_NAMES: Final = tuple(spec.catalog_name for spec in TOKEN_PRICED_MODELS) + ("whisper",) @@ -137,60 +40,74 @@ def _cost_map_entry(path: Path, catalog_name: str) -> dict[str, object]: return COST_MAP_ADAPTER.validate_json(path.read_bytes())[f"azure_ai/{catalog_name}"] +@pytest.mark.parametrize("catalog_name", CATALOG_NAMES) +def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None: + routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{catalog_name}") + assert (routed_model, provider) == (catalog_name, "azure_ai") + + @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_name_is_priced_and_routed(spec: TokenPricedCatalogModel) -> None: - routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{spec.catalog_name}") - assert (routed_model, provider) == (spec.catalog_name, "azure_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["litellm_provider"] == "azure_ai" - assert info["mode"] == spec.mode - assert info["input_cost_per_token"] == spec.input_cost_per_token - assert info["output_cost_per_token"] == spec.output_cost_per_token - assert info["cache_read_input_token_cost"] == spec.cache_read_input_token_cost - assert info["max_input_tokens"] == spec.max_input_tokens - assert info["max_output_tokens"] == spec.max_output_tokens - assert info["max_tokens"] == spec.max_output_tokens - for flag in spec.supported_flags: - assert info[flag] is True, flag +def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: + prompt_cost, completion_cost_usd = cost_per_token( + model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION + ) + assert prompt_cost == pytest.approx(spec.dollars_per_million_input) + assert completion_cost_usd == pytest.approx(spec.dollars_per_million_output) @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "spec", - [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], - ids=lambda spec: spec.catalog_name, -) -def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: - prompt_cost, completion_cost = cost_per_token( - model=f"azure_ai/{spec.catalog_name}", prompt_tokens=1_000_000, completion_tokens=1_000_000 +@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) +def test_azure_ai_catalog_name_prices_the_same_in_any_casing(spec: TokenPricedCatalogModel) -> None: + lowercase_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + upper_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0) + assert upper_cost == lowercase_cost + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES) +def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None: + uncached_prompt_cost, _ = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + cached_prompt_cost, _ = cost_per_token( + model=f"azure_ai/{catalog_name}", + prompt_tokens=A_MILLION, + completion_tokens=0, + cache_read_input_tokens=A_MILLION, ) - assert prompt_cost == pytest.approx(spec.input_cost_per_token * 1_000_000) - assert completion_cost == pytest.approx(spec.output_cost_per_token * 1_000_000) + assert uncached_prompt_cost > 0 + assert cached_prompt_cost == pytest.approx(uncached_prompt_cost) @pytest.mark.usefixtures("local_model_cost_map") def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: - routed_model, provider, _, _ = get_llm_provider(model="azure_ai/whisper") - assert (routed_model, provider) == ("whisper", "azure_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["mode"] == "audio_transcription" - assert info["input_cost_per_second"] == 0.0001 - assert info["output_cost_per_second"] == 0.0001 + transcription: Final = TranscriptionResponse(text="hello") + transcription._hidden_params = { # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter + "custom_llm_provider": "azure_ai", + "model": "azure_ai/whisper", + "audio_transcription_duration": 3600, + } + cost = completion_cost( + completion_response=transcription, + model="azure_ai/whisper", + custom_llm_provider="azure_ai", + call_type="atranscription", + ) + assert cost == pytest.approx(0.36) @pytest.mark.parametrize("catalog_name", CATALOG_NAMES) def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None: - main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", catalog_name) - backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json", catalog_name) + main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name) + backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name) - assert str(main_entry["source"]).startswith("https://azure.microsoft.com/en-us/pricing/details/") + assert str(main_entry["source"]).startswith(AZURE_PRICING_PREFIX) assert backup_entry == main_entry -@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_entry_carries_its_retirement_date(spec: TokenPricedCatalogModel) -> None: - entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", spec.catalog_name) - assert entry.get("deprecation_date") == spec.deprecation_date +def test_azure_ai_model_router_spellings_share_one_entry() -> None: + underscore_entry = _cost_map_entry(MAIN_COST_MAP, "model_router") + hyphen_entry = _cost_map_entry(MAIN_COST_MAP, "model-router") + + assert {k: v for k, v in underscore_entry.items() if k != "comment"} == { + k: v for k, v in hyphen_entry.items() if k != "comment" + } From 5706952588ee2b2445e864ce8a85af3339bb138b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:22:51 -0700 Subject: [PATCH 6/8] fix(azure_ai): drop gpt-chat-latest effort levels, test prices via calculator litellm's azure_ai config rejects reasoning_effort for gpt-chat-latest and Azure documents a fixed reasoning level for it, so the entry no longer advertises reasoning_effort_levels. The catalog metadata tests compare cost_per_token and the whisper transcription cost with the entry the calculator read instead of with list-price literals, the pattern #40195 removed --- ...odel_prices_and_context_window_backup.json | 3 - model_prices_and_context_window.json | 3 - ...azure_ai_foundry_catalog_model_metadata.py | 80 +++++++++---------- 3 files changed, 40 insertions(+), 46 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 674a8b98304..7e7d8a9e930 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3591,9 +3591,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "reasoning_effort_levels": [ - "medium" - ], "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 674a8b98304..7e7d8a9e930 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3591,9 +3591,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "reasoning_effort_levels": [ - "medium" - ], "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 19b082edd8a..84d5cd2a7d4 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -1,11 +1,10 @@ -from dataclasses import dataclass from pathlib import Path from typing import Final import pytest from pydantic import TypeAdapter -from litellm import completion_cost, cost_per_token +from litellm import completion_cost, cost_per_token, get_model_info from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.types.utils import TranscriptionResponse @@ -15,31 +14,39 @@ BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_windo COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) AZURE_PRICING_PREFIX: Final = "https://azure.microsoft.com/en-us/pricing/details/" A_MILLION: Final = 1_000_000 +AN_HOUR_IN_SECONDS: Final = 3600 - -@dataclass(frozen=True, slots=True) -class TokenPricedCatalogModel: - catalog_name: str - dollars_per_million_input: float - dollars_per_million_output: float - - -TOKEN_PRICED_MODELS: Final = ( - TokenPricedCatalogModel("gpt-chat-latest", 5.0, 30.0), - TokenPricedCatalogModel("codex-mini", 1.5, 6.0), - TokenPricedCatalogModel("model-router", 0.14, 0.0), - TokenPricedCatalogModel("cohere-command-a", 2.5, 10.0), - TokenPricedCatalogModel("grok-4-20-reasoning", 1.25, 2.5), - TokenPricedCatalogModel("grok-4-20-non-reasoning", 1.25, 2.5), +TOKEN_PRICED_NAMES: Final = ( + "gpt-chat-latest", + "codex-mini", + "model-router", + "cohere-command-a", + "grok-4-20-reasoning", + "grok-4-20-non-reasoning", ) GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") -CATALOG_NAMES: Final = tuple(spec.catalog_name for spec in TOKEN_PRICED_MODELS) + ("whisper",) +CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",) def _cost_map_entry(path: Path, catalog_name: str) -> dict[str, object]: return COST_MAP_ADAPTER.validate_json(path.read_bytes())[f"azure_ai/{catalog_name}"] +def _whisper_transcription_cost(duration_seconds: int) -> float: + transcription: Final = TranscriptionResponse(text="hello") + transcription._hidden_params = { # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter + "custom_llm_provider": "azure_ai", + "model": "azure_ai/whisper", + "audio_transcription_duration": duration_seconds, + } + return completion_cost( + completion_response=transcription, + model="azure_ai/whisper", + custom_llm_provider="azure_ai", + call_type="atranscription", + ) + + @pytest.mark.parametrize("catalog_name", CATALOG_NAMES) def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None: routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{catalog_name}") @@ -47,20 +54,22 @@ def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None: @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: +@pytest.mark.parametrize("catalog_name", TOKEN_PRICED_NAMES) +def test_azure_ai_catalog_name_charges_its_own_entry_per_token(catalog_name: str) -> None: + entry: Final = get_model_info(f"azure_ai/{catalog_name}") prompt_cost, completion_cost_usd = cost_per_token( - model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION + model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION ) - assert prompt_cost == pytest.approx(spec.dollars_per_million_input) - assert completion_cost_usd == pytest.approx(spec.dollars_per_million_output) + assert prompt_cost > 0 + assert prompt_cost == pytest.approx(A_MILLION * entry["input_cost_per_token"]) + assert completion_cost_usd == pytest.approx(A_MILLION * entry["output_cost_per_token"]) @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_name_prices_the_same_in_any_casing(spec: TokenPricedCatalogModel) -> None: - lowercase_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) - upper_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0) +@pytest.mark.parametrize("catalog_name", TOKEN_PRICED_NAMES) +def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str) -> None: + lowercase_cost = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + upper_cost = cost_per_token(model=f"azure_ai/{catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0) assert upper_cost == lowercase_cost @@ -80,19 +89,10 @@ def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalo @pytest.mark.usefixtures("local_model_cost_map") def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: - transcription: Final = TranscriptionResponse(text="hello") - transcription._hidden_params = { # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter - "custom_llm_provider": "azure_ai", - "model": "azure_ai/whisper", - "audio_transcription_duration": 3600, - } - cost = completion_cost( - completion_response=transcription, - model="azure_ai/whisper", - custom_llm_provider="azure_ai", - call_type="atranscription", - ) - assert cost == pytest.approx(0.36) + one_second_cost: Final = _whisper_transcription_cost(1) + one_hour_cost: Final = _whisper_transcription_cost(AN_HOUR_IN_SECONDS) + assert one_second_cost > 0 + assert one_hour_cost == pytest.approx(AN_HOUR_IN_SECONDS * one_second_cost) @pytest.mark.parametrize("catalog_name", CATALOG_NAMES) From 3cadf2f8f7120bf10409a353ef08e4cdc6f78b80 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:35:19 -0700 Subject: [PATCH 7/8] test(azure_ai): charge the router fee over cached prompt tokens too --- .../llms/azure_ai/test_azure_ai_cost_calculator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 7df14b91741..a43fc3332af 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -138,6 +138,18 @@ class TestAzureModelRouterFlatCost: assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) assert completion_cost_usd == 0.0 + def test_unmapped_router_deployment_name_charges_the_fee_over_cached_prompt_tokens_too(self) -> None: + usage = Usage( + prompt_tokens=2000, + completion_tokens=800, + total_tokens=2800, + cache_read_input_tokens=500, + cache_creation_input_tokens=200, + ) + prompt_cost, completion_cost_usd = cost_per_token(model="azure-model-router", usage=usage) + assert prompt_cost == pytest.approx(2000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 + def test_router_deployment_name_as_both_names_charges_the_fee_once(self) -> None: usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) prompt_cost, completion_cost_usd = cost_per_token( From 831a2a13fbd87aee9e77cc6831963ac4a9975fbb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:12:18 -0700 Subject: [PATCH 8/8] fix(azure): price azure_ai transcriptions at the azure_ai cost-map entry --- litellm/llms/azure/audio_transcriptions.py | 7 ++- litellm/main.py | 1 + .../llms/azure/test_audio_transcriptions.py | 61 +++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/azure/test_audio_transcriptions.py diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 4a5ed2ccb0c..564ec94ba6b 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -37,6 +37,7 @@ class AzureAudioTranscription(AzureChatCompletion): azure_ad_token: str | None = None, atranscription: bool = False, litellm_params: dict | None = None, + custom_llm_provider: str = "azure", ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} @@ -53,6 +54,7 @@ class AzureAudioTranscription(AzureChatCompletion): logging_obj=logging_obj, model=model, litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, ) azure_client: Final = self.get_azure_openai_client( @@ -99,7 +101,7 @@ class AzureAudioTranscription(AzureChatCompletion): additional_args={"complete_input_dict": data}, original_response=stringified_response, ) - hidden_params: Final = {"model": model, "custom_llm_provider": "azure"} + hidden_params: Final = {"model": model, "custom_llm_provider": custom_llm_provider} final_response: Final[TranscriptionResponse] = convert_to_model_response_object( response_object=stringified_response, model_response_object=model_response, @@ -122,6 +124,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=None, max_retries=None, litellm_params: dict | None = None, + custom_llm_provider: str = "azure", ) -> TranscriptionResponse: response = None try: @@ -178,7 +181,7 @@ class AzureAudioTranscription(AzureChatCompletion): }, original_response=stringified_response, ) - hidden_params: Final = {"model": model, "custom_llm_provider": "azure"} + hidden_params: Final = {"model": model, "custom_llm_provider": custom_llm_provider} response = convert_to_model_response_object( _response_headers=headers, response_object=stringified_response, diff --git a/litellm/main.py b/litellm/main.py index 75b7f7f10a5..11b30c240aa 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7805,6 +7805,7 @@ def transcription( azure_ad_token=azure_ad_token, max_retries=max_retries, litellm_params=litellm_params_dict, + custom_llm_provider=custom_llm_provider, ) elif custom_llm_provider == "openai" or (custom_llm_provider in litellm.openai_compatible_providers): api_base = ( diff --git a/tests/test_litellm/llms/azure/test_audio_transcriptions.py b/tests/test_litellm/llms/azure/test_audio_transcriptions.py new file mode 100644 index 00000000000..cd5fcbd85a9 --- /dev/null +++ b/tests/test_litellm/llms/azure/test_audio_transcriptions.py @@ -0,0 +1,61 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest +from openai import AzureOpenAI + +import litellm +from litellm.cost_calculator import completion_cost +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration + +AUDIO_FILE: Final = Path(__file__).parents[3] / "gettysburg.wav" +WHISPER_COST_PER_SECOND: Final = 0.0001 + + +def _transcription_client() -> AzureOpenAI: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"text": "Four score and seven years ago"}) + + return AzureOpenAI( + api_key="test-key", + api_version="2024-06-01", + azure_endpoint="https://example.cognitiveservices.azure.com", + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + + +def test_azure_ai_transcription_is_priced_at_the_azure_ai_entry(): + with AUDIO_FILE.open("rb") as audio: + response = litellm.transcription( + model="azure_ai/whisper", + file=audio, + api_base="https://example.cognitiveservices.azure.com", + api_key="test-key", + api_version="2024-06-01", + client=_transcription_client(), + ) + with AUDIO_FILE.open("rb") as audio: + duration = calculate_request_duration(audio) + + assert duration is not None and duration > 0 + assert response._hidden_params["custom_llm_provider"] == "azure_ai" + assert completion_cost(completion_response=response, call_type="transcription") == pytest.approx( + WHISPER_COST_PER_SECOND * duration + ) + + +def test_azure_transcription_keeps_the_azure_provider(): + with AUDIO_FILE.open("rb") as audio: + response = litellm.transcription( + model="azure/whisper-1", + file=audio, + api_base="https://example.openai.azure.com", + api_key="test-key", + api_version="2024-06-01", + client=_transcription_client(), + ) + + assert response._hidden_params["custom_llm_provider"] == "azure" + assert json.loads(response.model_dump_json())["text"] == "Four score and seven years ago"