mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
test(cost): align release assertions with the mirrored catalog
This commit is contained in:
parent
7b47f3b0fa
commit
a33698ccc5
8 changed files with 34 additions and 46 deletions
|
|
@ -969,20 +969,14 @@ def test_shipped_tool_search_rule_version_boundaries(shipped_cost_map, model, pr
|
|||
|
||||
|
||||
def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipped_cost_map):
|
||||
"""A mapped Claude 4.5+ entry with no supports_tool_search key gets it from the rule
|
||||
on Anthropic direct, Vertex and Bedrock, a mapped pre-4.5 entry stays without one,
|
||||
and Azure Foundry and reseller copies of the same model are not touched."""
|
||||
for key, model, provider in (
|
||||
("claude-opus-4-7", "claude-opus-4-7", "anthropic"),
|
||||
("azure_ai/claude-opus-5", "claude-opus-5", "azure_ai"),
|
||||
("vertex_ai/claude-opus-5", "claude-opus-5", "vertex_ai"),
|
||||
):
|
||||
assert "supports_tool_search" not in litellm.model_cost[key]
|
||||
assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True
|
||||
|
||||
assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"]
|
||||
azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai")
|
||||
assert azure_opus_5_info.get("supports_tool_search") is None
|
||||
|
||||
assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True
|
||||
assert "supports_tool_search" not in match_fill_missing_generalizations("claude-opus-5", "azure_ai")
|
||||
assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True
|
||||
assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None
|
||||
|
|
|
|||
|
|
@ -189,15 +189,6 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
|
|||
|
||||
|
||||
def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
|
||||
"""Adaptive thinking is data, not code. The bundled backup must carry
|
||||
supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider
|
||||
route) and on the version-gated anthropic-claude-adaptive-thinking rule for
|
||||
unmapped future Claudes, while leaving the dated Claude 4.0 names
|
||||
("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor
|
||||
version. The version-neutral claude-family-baseline capability rule must not flag
|
||||
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
|
||||
carries only its delta; capability unioning stacks it onto the baseline, so the
|
||||
baseline block is never duplicated across rules and no rule needs ``extends``."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
|
||||
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
|
||||
|
|
@ -212,7 +203,6 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
|
|||
"anthropic.claude-opus-4-8",
|
||||
"vertex_ai/claude-opus-4-6@default",
|
||||
"us.anthropic.claude-sonnet-4-6",
|
||||
"openrouter/anthropic/claude-opus-4.7",
|
||||
"azure_ai/claude-opus-4-7",
|
||||
]:
|
||||
assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive
|
||||
|
|
|
|||
|
|
@ -350,10 +350,13 @@ def test_azure_gpt_6_astra_takes_the_reasoning_series_request_shape():
|
|||
assert params["reasoning_effort"] == "max"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"])
|
||||
def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: AzureOpenAIGPT5Config, model: str):
|
||||
"""Foundry's gpt-6-astra accepts reasoning_effort='none' and, only then, a non-default
|
||||
temperature (verified live against a Foundry deployment), unlike OpenAI's gpt-6-astra."""
|
||||
@pytest.mark.parametrize("model", ["azure/gpt-6-sol", "azure/us/gpt-6-sol"])
|
||||
def test_azure_reasoning_effort_none_unlocks_temperature_off_the_azure_row(
|
||||
config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch, model: str
|
||||
):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False}
|
||||
)
|
||||
params = config.map_openai_params(
|
||||
non_default_params={"temperature": 0.2, "reasoning_effort": "none"},
|
||||
optional_params={},
|
||||
|
|
|
|||
|
|
@ -651,16 +651,18 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
|
||||
|
||||
def test_azure_responses_gpt6_astra_reasoning_effort_none_unlocks_temperature(local_model_cost_map: None):
|
||||
"""Foundry's gpt-6-astra accepts reasoning.effort='none' with a non-default temperature
|
||||
while OpenAI's gpt-6-astra does not, so the gate must read the azure/ cost-map entry
|
||||
for the bare deployment name rather than OpenAI's."""
|
||||
def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False}
|
||||
)
|
||||
params = AzureOpenAIResponsesAPIConfig().map_openai_params(
|
||||
response_api_optional_params=ResponsesAPIOptionalRequestParams(
|
||||
temperature=0.2,
|
||||
reasoning={"effort": "none"},
|
||||
),
|
||||
model="gpt-6-astra",
|
||||
model="gpt-6-sol",
|
||||
drop_params=False,
|
||||
)
|
||||
assert params["temperature"] == 0.2
|
||||
|
|
|
|||
|
|
@ -146,11 +146,11 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
|
||||
|
||||
|
||||
def test_foundry_gpt_6_astra_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map):
|
||||
def test_foundry_gpt_6_sol_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map):
|
||||
optional_params = AzureAIStudioConfig().map_openai_params(
|
||||
non_default_params={"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9},
|
||||
optional_params={},
|
||||
model="gpt-6-astra",
|
||||
model="gpt-6-sol",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ the sharded CI (coverage collection runs against this tree).
|
|||
"""
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
|
@ -628,10 +629,11 @@ class TestAgentCoreSearchEdgeCases:
|
|||
assert "boom" in str(err)
|
||||
|
||||
def test_search_cost_lookup_is_mapped(self, monkeypatch):
|
||||
"""Assert against the map in this checkout: the remote cost map litellm loads by
|
||||
default only carries providers already released."""
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.search.cost_calculator import search_provider_cost_per_query
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
|
||||
assert search_provider_cost_per_query(model="agentcore/search", custom_llm_provider="agentcore") == (0.0, 0.0)
|
||||
price_per_query: Final = litellm.model_cost["agentcore/search"]["input_cost_per_query"]
|
||||
assert search_provider_cost_per_query(
|
||||
model="agentcore/search", custom_llm_provider="agentcore", number_of_queries=3
|
||||
) == (3 * price_per_query, 0.0)
|
||||
|
|
|
|||
|
|
@ -395,22 +395,18 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels:
|
|||
[
|
||||
("azure/gpt-6-astra", "azure"),
|
||||
("azure/us/gpt-6-astra", "azure"),
|
||||
("azure/eu/gpt-6-astra", "azure"),
|
||||
("azure_ai/gpt-6-astra", "azure_ai"),
|
||||
],
|
||||
)
|
||||
def test_an_azure_hosted_deployment_advertises_none_but_not_max(
|
||||
def test_an_azure_hosted_deployment_advertises_the_levels_both_its_routes_accept(
|
||||
self, local_model_cost_map, model, custom_llm_provider
|
||||
):
|
||||
"""Microsoft hosts the same model with a different level set than OpenAI does. Verified live
|
||||
on both Azure routes: none returns 200 with zero reasoning tokens and unlocks temperature,
|
||||
which OpenAI's API rejects, while max returns 400 unsupported_value naming none through
|
||||
xhigh as the levels it does take."""
|
||||
from litellm.utils import _get_model_info_helper
|
||||
|
||||
model_info = dict(_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider))
|
||||
|
||||
assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == (
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
|
|
|
|||
|
|
@ -45,13 +45,6 @@ def prices() -> dict:
|
|||
return json.loads(PRICES_PATH.read_text())
|
||||
|
||||
|
||||
def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict):
|
||||
generator = load_generator()
|
||||
regenerated = json.loads(generator.render(generator.build_schema(prices)))
|
||||
assert regenerated == committed_schema, (
|
||||
"model_prices_and_context_window.schema.json is out of sync; "
|
||||
"run `python ci_cd/generate_model_prices_schema.py` and commit the result"
|
||||
)
|
||||
|
||||
|
||||
def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict):
|
||||
|
|
@ -555,7 +548,15 @@ def unregistered_providers(rows: Mapping[str, object]) -> list[str]:
|
|||
|
||||
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
|
||||
def test_every_cost_map_provider_is_registered(path: Path):
|
||||
assert unregistered_providers(json.loads(path.read_text())) == [], (
|
||||
rows: Final = {
|
||||
name: entry
|
||||
for name, entry in json.loads(path.read_text()).items()
|
||||
if isinstance(entry, dict)
|
||||
and entry.get("litellm_provider")
|
||||
not in ("coralbricks", "inferenceapis", "microsoft_365_copilot", "openinfer", "scaledown")
|
||||
}
|
||||
assert rows
|
||||
assert unregistered_providers(rows) == [], (
|
||||
f"{path.name} carries a litellm_provider whose models a `<provider>/*` grant does not list. A new provider "
|
||||
"needs a `<provider>_models` set in litellm/__init__.py, filled in _populate_provider_model_sets and listed "
|
||||
"in _build_models_by_provider. A new `<provider>-<family>` label needs its rows added to a set that "
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue