mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
test(cost): keep provider exemptions explicit on the release
This commit is contained in:
parent
945075d627
commit
6ceb19c2e9
3 changed files with 21 additions and 17 deletions
|
|
@ -393,25 +393,32 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels:
|
|||
@pytest.mark.parametrize(
|
||||
"model,custom_llm_provider",
|
||||
[
|
||||
("azure/gpt-6-sol", "azure"),
|
||||
("azure/gpt-6-luna", "azure"),
|
||||
("azure/eu/gpt-6-sol", "azure"),
|
||||
("azure/eu/gpt-6-luna", "azure"),
|
||||
("azure_ai/gpt-6-sol", "azure_ai"),
|
||||
("azure_ai/gpt-6-luna", "azure_ai"),
|
||||
("azure/gpt-6-astra", "azure"),
|
||||
("azure/us/gpt-6-astra", "azure"),
|
||||
("azure/eu/gpt-6-astra", "azure"),
|
||||
("azure_ai/gpt-6-astra", "azure_ai"),
|
||||
],
|
||||
)
|
||||
def test_the_azure_entry_advertises_the_same_levels_as_openai(
|
||||
def test_an_azure_hosted_deployment_advertises_the_levels_both_its_routes_accept(
|
||||
self, local_model_cost_map, model, custom_llm_provider
|
||||
):
|
||||
"""Verified live on 2026-10-01 against a Foundry gpt-6-astra deployment (model
|
||||
gpt-6-astra-2026-09-03): none returns 400 unsupported_value on chat and on Responses, naming
|
||||
low, medium, high and xhigh as the levels it takes; Responses also honors max with reasoning
|
||||
tokens billed while chat refuses it, as OpenAI's chat does. The flag is route-blind, so the
|
||||
row keeps max off and the advertised set stops at the level every route accepts.
|
||||
Earlier evidence (2026-09-05, commit e79f3ec5205) had none accepted on chat; the deployment
|
||||
no longer behaves that way."""
|
||||
from litellm.utils import get_model_info_helper
|
||||
|
||||
azure_info = dict(get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider))
|
||||
openai_info = dict(get_model_info_helper(model=model.rsplit("/", 1)[1], custom_llm_provider="openai"))
|
||||
model_info = dict(get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider))
|
||||
|
||||
assert resolve_supported_reasoning_efforts(
|
||||
azure_info, deployment_is_mapped=True
|
||||
) == resolve_supported_reasoning_efforts(openai_info, deployment_is_mapped=True)
|
||||
assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == (
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
)
|
||||
|
||||
|
||||
class TestGpt6SolAndLunaAdvertiseNoneThroughMax:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from litellm.litellm_core_utils.fallback_generalizations import match_capability
|
|||
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
|
|
@ -679,12 +678,10 @@ def unregistered_providers(rows: Mapping[str, object]) -> list[str]:
|
|||
|
||||
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
|
||||
def test_every_cost_map_provider_is_registered(path: Path):
|
||||
supported_providers: Final = frozenset(provider.value for provider in LlmProviders)
|
||||
rows: Final = {
|
||||
name: entry
|
||||
for name, entry in json.loads(path.read_text()).items()
|
||||
if isinstance(entry, dict)
|
||||
and entry.get("litellm_provider", "").split("-", 1)[0] in supported_providers
|
||||
if isinstance(entry, dict) and entry.get("litellm_provider") != "openinfer"
|
||||
}
|
||||
assert rows
|
||||
assert unregistered_providers(rows) == [], (
|
||||
|
|
|
|||
|
|
@ -8561,7 +8561,7 @@ def test_models_by_provider():
|
|||
continue
|
||||
elif k == "sample_spec":
|
||||
continue
|
||||
elif v["litellm_provider"] not in litellm.provider_list and not JSONProviderRegistry.exists(v["litellm_provider"]):
|
||||
elif v["litellm_provider"] == "openinfer":
|
||||
continue
|
||||
elif (
|
||||
v["litellm_provider"] == "sagemaker"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue