From 6ceb19c2e99bc4227a30fd847e78dc57356c3af8 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:00:21 -0700 Subject: [PATCH] test(cost): keep provider exemptions explicit on the release --- .../test_reasoning_effort_capability.py | 31 ++++++++++++------- .../test_model_prices_and_context_window.py | 5 +-- tests/unit/test_utils.py | 2 +- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 3f06d356dc8..2226b1f39df 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -393,25 +393,32 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: @pytest.mark.parametrize( "model,custom_llm_provider", [ - ("azure/gpt-6-sol", "azure"), - ("azure/gpt-6-luna", "azure"), - ("azure/eu/gpt-6-sol", "azure"), - ("azure/eu/gpt-6-luna", "azure"), - ("azure_ai/gpt-6-sol", "azure_ai"), - ("azure_ai/gpt-6-luna", "azure_ai"), + ("azure/gpt-6-astra", "azure"), + ("azure/us/gpt-6-astra", "azure"), + ("azure/eu/gpt-6-astra", "azure"), + ("azure_ai/gpt-6-astra", "azure_ai"), ], ) - def test_the_azure_entry_advertises_the_same_levels_as_openai( + def test_an_azure_hosted_deployment_advertises_the_levels_both_its_routes_accept( self, local_model_cost_map, model, custom_llm_provider ): + """Verified live on 2026-10-01 against a Foundry gpt-6-astra deployment (model + gpt-6-astra-2026-09-03): none returns 400 unsupported_value on chat and on Responses, naming + low, medium, high and xhigh as the levels it takes; Responses also honors max with reasoning + tokens billed while chat refuses it, as OpenAI's chat does. The flag is route-blind, so the + row keeps max off and the advertised set stops at the level every route accepts. + Earlier evidence (2026-09-05, commit e79f3ec5205) had none accepted on chat; the deployment + no longer behaves that way.""" from litellm.utils import get_model_info_helper - azure_info = dict(get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)) - openai_info = dict(get_model_info_helper(model=model.rsplit("/", 1)[1], custom_llm_provider="openai")) + model_info = dict(get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)) - assert resolve_supported_reasoning_efforts( - azure_info, deployment_is_mapped=True - ) == resolve_supported_reasoning_efforts(openai_info, deployment_is_mapped=True) + assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( + "low", + "medium", + "high", + "xhigh", + ) class TestGpt6SolAndLunaAdvertiseNoneThroughMax: diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index d9d465664ce..e33d3074046 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -16,7 +16,6 @@ from litellm.litellm_core_utils.fallback_generalizations import match_capability from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts -from litellm.types.utils import LlmProviders REPO_ROOT = Path(__file__).parents[2] PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" @@ -679,12 +678,10 @@ def unregistered_providers(rows: Mapping[str, object]) -> list[str]: @pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) def test_every_cost_map_provider_is_registered(path: Path): - supported_providers: Final = frozenset(provider.value for provider in LlmProviders) rows: Final = { name: entry for name, entry in json.loads(path.read_text()).items() - if isinstance(entry, dict) - and entry.get("litellm_provider", "").split("-", 1)[0] in supported_providers + if isinstance(entry, dict) and entry.get("litellm_provider") != "openinfer" } assert rows assert unregistered_providers(rows) == [], ( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6cbf51a2b59..6c1876ae510 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8561,7 +8561,7 @@ def test_models_by_provider(): continue elif k == "sample_spec": continue - elif v["litellm_provider"] not in litellm.provider_list and not JSONProviderRegistry.exists(v["litellm_provider"]): + elif v["litellm_provider"] == "openinfer": continue elif ( v["litellm_provider"] == "sagemaker"