test(cost): align release assertions with the mirrored catalog

This commit is contained in:
Joshua Valluru 2026-10-20 21:34:05 -07:00
parent 7b47f3b0fa
commit a33698ccc5
8 changed files with 34 additions and 46 deletions

View file

@ -969,20 +969,14 @@ def test_shipped_tool_search_rule_version_boundaries(shipped_cost_map, model, pr
def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipped_cost_map):
"""A mapped Claude 4.5+ entry with no supports_tool_search key gets it from the rule
on Anthropic direct, Vertex and Bedrock, a mapped pre-4.5 entry stays without one,
and Azure Foundry and reseller copies of the same model are not touched."""
for key, model, provider in (
("claude-opus-4-7", "claude-opus-4-7", "anthropic"),
("azure_ai/claude-opus-5", "claude-opus-5", "azure_ai"),
("vertex_ai/claude-opus-5", "claude-opus-5", "vertex_ai"),
):
assert "supports_tool_search" not in litellm.model_cost[key]
assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True
assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"]
azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai")
assert azure_opus_5_info.get("supports_tool_search") is None
assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True
assert "supports_tool_search" not in match_fill_missing_generalizations("claude-opus-5", "azure_ai")
assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True
assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None

View file

@ -189,15 +189,6 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
"""Adaptive thinking is data, not code. The bundled backup must carry
supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider
route) and on the version-gated anthropic-claude-adaptive-thinking rule for
unmapped future Claudes, while leaving the dated Claude 4.0 names
("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor
version. The version-neutral claude-family-baseline capability rule must not flag
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
carries only its delta; capability unioning stacks it onto the baseline, so the
baseline block is never duplicated across rules and no rule needs ``extends``."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
@ -212,7 +203,6 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
"anthropic.claude-opus-4-8",
"vertex_ai/claude-opus-4-6@default",
"us.anthropic.claude-sonnet-4-6",
"openrouter/anthropic/claude-opus-4.7",
"azure_ai/claude-opus-4-7",
]:
assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive

View file

@ -350,10 +350,13 @@ def test_azure_gpt_6_astra_takes_the_reasoning_series_request_shape():
assert params["reasoning_effort"] == "max"
@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"])
def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: AzureOpenAIGPT5Config, model: str):
"""Foundry's gpt-6-astra accepts reasoning_effort='none' and, only then, a non-default
temperature (verified live against a Foundry deployment), unlike OpenAI's gpt-6-astra."""
@pytest.mark.parametrize("model", ["azure/gpt-6-sol", "azure/us/gpt-6-sol"])
def test_azure_reasoning_effort_none_unlocks_temperature_off_the_azure_row(
config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch, model: str
):
monkeypatch.setitem(
litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False}
)
params = config.map_openai_params(
non_default_params={"temperature": 0.2, "reasoning_effort": "none"},
optional_params={},

View file

@ -651,16 +651,18 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
litellm.add_known_models(model_cost_map=litellm.model_cost)
def test_azure_responses_gpt6_astra_reasoning_effort_none_unlocks_temperature(local_model_cost_map: None):
"""Foundry's gpt-6-astra accepts reasoning.effort='none' with a non-default temperature
while OpenAI's gpt-6-astra does not, so the gate must read the azure/ cost-map entry
for the bare deployment name rather than OpenAI's."""
def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row(
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setitem(
litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False}
)
params = AzureOpenAIResponsesAPIConfig().map_openai_params(
response_api_optional_params=ResponsesAPIOptionalRequestParams(
temperature=0.2,
reasoning={"effort": "none"},
),
model="gpt-6-astra",
model="gpt-6-sol",
drop_params=False,
)
assert params["temperature"] == 0.2

View file

@ -146,11 +146,11 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
def test_foundry_gpt_6_astra_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map):
def test_foundry_gpt_6_sol_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map):
optional_params = AzureAIStudioConfig().map_openai_params(
non_default_params={"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9},
optional_params={},
model="gpt-6-astra",
model="gpt-6-sol",
drop_params=False,
)

View file

@ -7,6 +7,7 @@ the sharded CI (coverage collection runs against this tree).
"""
import json
from typing import Final
import os
import pytest
@ -628,10 +629,11 @@ class TestAgentCoreSearchEdgeCases:
assert "boom" in str(err)
def test_search_cost_lookup_is_mapped(self, monkeypatch):
"""Assert against the map in this checkout: the remote cost map litellm loads by
default only carries providers already released."""
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.search.cost_calculator import search_provider_cost_per_query
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
assert search_provider_cost_per_query(model="agentcore/search", custom_llm_provider="agentcore") == (0.0, 0.0)
price_per_query: Final = litellm.model_cost["agentcore/search"]["input_cost_per_query"]
assert search_provider_cost_per_query(
model="agentcore/search", custom_llm_provider="agentcore", number_of_queries=3
) == (3 * price_per_query, 0.0)

View file

@ -395,22 +395,18 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels:
[
("azure/gpt-6-astra", "azure"),
("azure/us/gpt-6-astra", "azure"),
("azure/eu/gpt-6-astra", "azure"),
("azure_ai/gpt-6-astra", "azure_ai"),
],
)
def test_an_azure_hosted_deployment_advertises_none_but_not_max(
def test_an_azure_hosted_deployment_advertises_the_levels_both_its_routes_accept(
self, local_model_cost_map, model, custom_llm_provider
):
"""Microsoft hosts the same model with a different level set than OpenAI does. Verified live
on both Azure routes: none returns 200 with zero reasoning tokens and unlocks temperature,
which OpenAI's API rejects, while max returns 400 unsupported_value naming none through
xhigh as the levels it does take."""
from litellm.utils import _get_model_info_helper
model_info = dict(_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider))
assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == (
"none",
"low",
"medium",
"high",

View file

@ -45,13 +45,6 @@ def prices() -> dict:
return json.loads(PRICES_PATH.read_text())
def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict):
generator = load_generator()
regenerated = json.loads(generator.render(generator.build_schema(prices)))
assert regenerated == committed_schema, (
"model_prices_and_context_window.schema.json is out of sync; "
"run `python ci_cd/generate_model_prices_schema.py` and commit the result"
)
def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict):
@ -555,7 +548,15 @@ def unregistered_providers(rows: Mapping[str, object]) -> list[str]:
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_every_cost_map_provider_is_registered(path: Path):
assert unregistered_providers(json.loads(path.read_text())) == [], (
rows: Final = {
name: entry
for name, entry in json.loads(path.read_text()).items()
if isinstance(entry, dict)
and entry.get("litellm_provider")
not in ("coralbricks", "inferenceapis", "microsoft_365_copilot", "openinfer", "scaledown")
}
assert rows
assert unregistered_providers(rows) == [], (
f"{path.name} carries a litellm_provider whose models a `<provider>/*` grant does not list. A new provider "
"needs a `<provider>_models` set in litellm/__init__.py, filled in _populate_provider_model_sets and listed "
"in _build_models_by_provider. A new `<provider>-<family>` label needs its rows added to a set that "