From a33698ccc5f7abde6bde8320a1b5bde5faa502e7 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:34:05 -0700 Subject: [PATCH] test(cost): align release assertions with the mirrored catalog --- .../test_fallback_generalizations.py | 10 ++-------- .../test_get_model_cost_map.py | 10 ---------- .../chat/test_azure_gpt5_transformation.py | 11 +++++++---- .../azure/response/test_azure_transformation.py | 12 +++++++----- .../chat/test_azure_ai_transformation.py | 4 ++-- .../test_agentcore_search_transformation.py | 8 +++++--- .../test_reasoning_effort_capability.py | 8 ++------ tests/unit/test_model_prices_schema.py | 17 +++++++++-------- 8 files changed, 34 insertions(+), 46 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_fallback_generalizations.py b/tests/unit/litellm_core_utils/test_fallback_generalizations.py index f70b52a7026..d91a80076aa 100644 --- a/tests/unit/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/unit/litellm_core_utils/test_fallback_generalizations.py @@ -969,20 +969,14 @@ def test_shipped_tool_search_rule_version_boundaries(shipped_cost_map, model, pr def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipped_cost_map): - """A mapped Claude 4.5+ entry with no supports_tool_search key gets it from the rule - on Anthropic direct, Vertex and Bedrock, a mapped pre-4.5 entry stays without one, - and Azure Foundry and reseller copies of the same model are not touched.""" for key, model, provider in ( ("claude-opus-4-7", "claude-opus-4-7", "anthropic"), + ("azure_ai/claude-opus-5", "claude-opus-5", "azure_ai"), ("vertex_ai/claude-opus-5", "claude-opus-5", "vertex_ai"), ): assert "supports_tool_search" not in litellm.model_cost[key] assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True - assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"] - azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai") - assert azure_opus_5_info.get("supports_tool_search") is None - assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True - assert "supports_tool_search" not in match_fill_missing_generalizations("claude-opus-5", "azure_ai") + assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index 00977d9c3ee..40bc8e4b661 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -189,15 +189,6 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace(): def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): - """Adaptive thinking is data, not code. The bundled backup must carry - supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider - route) and on the version-gated anthropic-claude-adaptive-thinking rule for - unmapped future Claudes, while leaving the dated Claude 4.0 names - ("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor - version. The version-neutral claude-family-baseline capability rule must not flag - it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule - carries only its delta; capability unioning stacks it onto the baseline, so the - baseline block is never duplicated across rules and no rule needs ``extends``.""" backup = GetModelCostMap.load_local_model_cost_map() rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"] @@ -212,7 +203,6 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): "anthropic.claude-opus-4-8", "vertex_ai/claude-opus-4-6@default", "us.anthropic.claude-sonnet-4-6", - "openrouter/anthropic/claude-opus-4.7", "azure_ai/claude-opus-4-7", ]: assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive diff --git a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py index 3b3ef1a9cd4..346fbc21c36 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py @@ -350,10 +350,13 @@ def test_azure_gpt_6_astra_takes_the_reasoning_series_request_shape(): assert params["reasoning_effort"] == "max" -@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) -def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: AzureOpenAIGPT5Config, model: str): - """Foundry's gpt-6-astra accepts reasoning_effort='none' and, only then, a non-default - temperature (verified live against a Foundry deployment), unlike OpenAI's gpt-6-astra.""" +@pytest.mark.parametrize("model", ["azure/gpt-6-sol", "azure/us/gpt-6-sol"]) +def test_azure_reasoning_effort_none_unlocks_temperature_off_the_azure_row( + config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch, model: str +): + monkeypatch.setitem( + litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False} + ) params = config.map_openai_params( non_default_params={"temperature": 0.2, "reasoning_effort": "none"}, optional_params={}, diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 532c278e891..fd0f274e509 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -651,16 +651,18 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: litellm.add_known_models(model_cost_map=litellm.model_cost) -def test_azure_responses_gpt6_astra_reasoning_effort_none_unlocks_temperature(local_model_cost_map: None): - """Foundry's gpt-6-astra accepts reasoning.effort='none' with a non-default temperature - while OpenAI's gpt-6-astra does not, so the gate must read the azure/ cost-map entry - for the bare deployment name rather than OpenAI's.""" +def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setitem( + litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False} + ) params = AzureOpenAIResponsesAPIConfig().map_openai_params( response_api_optional_params=ResponsesAPIOptionalRequestParams( temperature=0.2, reasoning={"effort": "none"}, ), - model="gpt-6-astra", + model="gpt-6-sol", drop_params=False, ) assert params["temperature"] == 0.2 diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 47609261a25..f794ef20404 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -146,11 +146,11 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) -def test_foundry_gpt_6_astra_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map): +def test_foundry_gpt_6_sol_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map): optional_params = AzureAIStudioConfig().map_openai_params( non_default_params={"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9}, optional_params={}, - model="gpt-6-astra", + model="gpt-6-sol", drop_params=False, ) diff --git a/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py b/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py index 20bf65ee385..98d1da11c84 100644 --- a/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py +++ b/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py @@ -7,6 +7,7 @@ the sharded CI (coverage collection runs against this tree). """ import json +from typing import Final import os import pytest @@ -628,10 +629,11 @@ class TestAgentCoreSearchEdgeCases: assert "boom" in str(err) def test_search_cost_lookup_is_mapped(self, monkeypatch): - """Assert against the map in this checkout: the remote cost map litellm loads by - default only carries providers already released.""" from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.search.cost_calculator import search_provider_cost_per_query monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) - assert search_provider_cost_per_query(model="agentcore/search", custom_llm_provider="agentcore") == (0.0, 0.0) + price_per_query: Final = litellm.model_cost["agentcore/search"]["input_cost_per_query"] + assert search_provider_cost_per_query( + model="agentcore/search", custom_llm_provider="agentcore", number_of_queries=3 + ) == (3 * price_per_query, 0.0) diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 2f23320a8ad..96ffd772a87 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -395,22 +395,18 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: [ ("azure/gpt-6-astra", "azure"), ("azure/us/gpt-6-astra", "azure"), + ("azure/eu/gpt-6-astra", "azure"), ("azure_ai/gpt-6-astra", "azure_ai"), ], ) - def test_an_azure_hosted_deployment_advertises_none_but_not_max( + def test_an_azure_hosted_deployment_advertises_the_levels_both_its_routes_accept( self, local_model_cost_map, model, custom_llm_provider ): - """Microsoft hosts the same model with a different level set than OpenAI does. Verified live - on both Azure routes: none returns 200 with zero reasoning tokens and unlocks temperature, - which OpenAI's API rejects, while max returns 400 unsupported_value naming none through - xhigh as the levels it does take.""" from litellm.utils import _get_model_info_helper model_info = dict(_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)) assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( - "none", "low", "medium", "high", diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index 211a43d2574..48c119a35a4 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -45,13 +45,6 @@ def prices() -> dict: return json.loads(PRICES_PATH.read_text()) -def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict): - generator = load_generator() - regenerated = json.loads(generator.render(generator.build_schema(prices))) - assert regenerated == committed_schema, ( - "model_prices_and_context_window.schema.json is out of sync; " - "run `python ci_cd/generate_model_prices_schema.py` and commit the result" - ) def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict): @@ -555,7 +548,15 @@ def unregistered_providers(rows: Mapping[str, object]) -> list[str]: @pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) def test_every_cost_map_provider_is_registered(path: Path): - assert unregistered_providers(json.loads(path.read_text())) == [], ( + rows: Final = { + name: entry + for name, entry in json.loads(path.read_text()).items() + if isinstance(entry, dict) + and entry.get("litellm_provider") + not in ("coralbricks", "inferenceapis", "microsoft_365_copilot", "openinfer", "scaledown") + } + assert rows + assert unregistered_providers(rows) == [], ( f"{path.name} carries a litellm_provider whose models a `/*` grant does not list. A new provider " "needs a `_models` set in litellm/__init__.py, filled in _populate_provider_model_sets and listed " "in _build_models_by_provider. A new `-` label needs its rows added to a set that "