From 044f88ee91942d60cdd4cc8b060b1e95fc60cb0c Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 18 Sep 2026 21:04:25 -0700 Subject: [PATCH 1/3] fix(proxy): register transcribe as a known provider for model grants #41515 added the cost map entry transcribe/StartTranscriptionJob under a new litellm_provider value "transcribe" without registering that provider anywhere else, so litellm.models_by_provider had no "transcribe" key. test_models_by_provider derives its provider set from the cost map itself, so it went red on main. The user-visible half is that get_provider_models returned None for the provider, which get_known_models_from_wildcard turns into an empty list, leaving a transcribe/* key or team grant resolving to no models. Mirror the aws_polly registration: an enum member, a model set, an ingestion branch, and a models_by_provider entry. Amazon Transcribe is reached through the pass-through route rather than the Add Model form, so it joins the frozen unlisted set the Add Model drift test tracks. --- litellm/__init__.py | 4 ++++ litellm/types/utils.py | 1 + .../test_litellm/proxy/auth/test_model_checks.py | 15 +++++++++++++++ .../public_endpoints/test_public_endpoints.py | 1 + 4 files changed, 21 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index fcfc4768ff3..e17ab613dac 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -701,6 +701,7 @@ github_copilot_models: Set = set() chatgpt_models: Set = set() minimax_models: Set = set() aws_polly_models: Set = set() +transcribe_models: Set = set() gigachat_models: Set = set() llamagate_models: Set = set() reducto_models: Set = set() @@ -980,6 +981,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: minimax_models.add(key) elif value.get("litellm_provider") == "aws_polly": aws_polly_models.add(key) + elif value.get("litellm_provider") == "transcribe": + transcribe_models.add(key) elif value.get("litellm_provider") == "gigachat": gigachat_models.add(key) elif value.get("litellm_provider") == "llamagate": @@ -1227,6 +1230,7 @@ def _build_models_by_provider() -> dict: "chatgpt": chatgpt_models, "minimax": minimax_models, "aws_polly": aws_polly_models, + "transcribe": transcribe_models, "gigachat": gigachat_models, "llamagate": llamagate_models, "reducto": reducto_models, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c63d971b89b..dbd293a7dc8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3989,6 +3989,7 @@ class LlmProviders(str, Enum): REDUCTO = "reducto" RUNWAYML = "runwayml" AWS_POLLY = "aws_polly" + TRANSCRIBE = "transcribe" HUGGINGFACE = "huggingface" TOGETHER_AI = "together_ai" OPENROUTER = "openrouter" diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 3c6733cb86d..f10622e954b 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -879,3 +879,18 @@ def test_get_complete_model_list_sentinel_only_grants_nothing(): infer_model_from_keys=False, ) assert result == [] + + +def test_transcribe_is_a_known_provider_for_wildcard_expansion(): + import litellm + from litellm.proxy.auth.model_checks import ( + get_known_models_from_wildcard, + get_provider_models, + ) + + assert "transcribe" in litellm.models_by_provider + assert "transcribe/StartTranscriptionJob" in litellm.models_by_provider["transcribe"] + assert get_provider_models("transcribe") == ["transcribe/StartTranscriptionJob"] + assert get_known_models_from_wildcard("transcribe/*") == [ + "transcribe/StartTranscriptionJob" + ] diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0d82ed778f5..aaa3b205312 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -384,6 +384,7 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "tencent", "tensormesh", "text-completion-inception", + "transcribe", "valkey", "xiaomi_mimo", "zai", From 6e0356a7a0fa180f11e1e038022b4dfe204e28a3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 19 Sep 2026 02:34:27 -0700 Subject: [PATCH 2/3] test: fail a required shard when a cost map provider is unregistered --- .../test_litellm/test_model_prices_schema.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index a9015397b32..fa42c65fb9a 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -11,7 +11,9 @@ from typing import Final import jsonschema import pytest +import litellm from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name +from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts REPO_ROOT = Path(__file__).parents[2] @@ -363,3 +365,50 @@ def test_active_mistral_chat_rows_price_cache_reads_below_input(path: Path): and not cache_read_is_tenth_of_input(entry) ] assert drifted == [] + + +PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"}) +MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"}) + + +def is_registered_provider(label: str) -> bool: + family_root: Final = label.split("-", 1)[0] + return any( + name in litellm.models_by_provider or JSONProviderRegistry.exists(name) for name in (label, family_root) + ) + + +def unregistered_providers(rows: Mapping[str, object]) -> list[str]: + return sorted( + { + entry["litellm_provider"] + for name, entry in rows.items() + if name != "sample_spec" + and isinstance(entry, dict) + and "litellm_provider" in entry + and entry.get("mode") not in MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY + and entry["litellm_provider"] not in PROVIDER_LABELS_WITHOUT_A_MODEL_SET + and not is_registered_provider(entry["litellm_provider"]) + } + ) + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_every_cost_map_provider_is_registered(path: Path): + assert unregistered_providers(json.loads(path.read_text())) == [], ( + f"{path.name} carries a litellm_provider that litellm.models_by_provider does not know, so a `/*` " + "grant expands to no models. Add a `_models` set in litellm/__init__.py, fill it in " + "_populate_provider_model_sets, and list it in _build_models_by_provider" + ) + + +def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): + rows: Final = { + "sample_spec": {"litellm_provider": "one of the supported providers", "mode": "chat"}, + "nobody_registered/StartJob": {"litellm_provider": "nobody_registered", "mode": "audio_transcription"}, + "gpt-4o": {"litellm_provider": "openai", "mode": "chat"}, + "vertex_ai/new-family-model": {"litellm_provider": "vertex_ai-new_family_models", "mode": "chat"}, + "unknown_root/model": {"litellm_provider": "unknown_root-new_family_models", "mode": "chat"}, + "some_search/search": {"litellm_provider": "some_search", "mode": "search"}, + } + assert unregistered_providers(rows) == ["nobody_registered", "unknown_root-new_family_models"] From 988bb65aa236393c81a3b267882d538376cc04b9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 19 Sep 2026 02:50:31 -0700 Subject: [PATCH 3/3] test: require a provider family's rows to reach its wildcard list --- .../test_litellm/test_model_prices_schema.py | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index fa42c65fb9a..918aff806c1 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -369,46 +369,71 @@ def test_active_mistral_chat_rows_price_cache_reads_below_input(path: Path): PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"}) MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"}) +VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset( + { + "vertex_ai-ai21_models", + "vertex_ai-embedding-models", + "vertex_ai-image-models", + "vertex_ai-llama_models", + "vertex_ai-mistral_models", + "vertex_ai-openai_models", + "vertex_ai-qwen_models", + "vertex_ai-video-models", + } +) -def is_registered_provider(label: str) -> bool: +def is_registered_provider(label: str, model_names: tuple[str, ...]) -> bool: + if label in litellm.models_by_provider or JSONProviderRegistry.exists(label): + return True family_root: Final = label.split("-", 1)[0] + wildcard_models: Final = litellm.models_by_provider.get(family_root, ()) return any( - name in litellm.models_by_provider or JSONProviderRegistry.exists(name) for name in (label, family_root) + name in wildcard_models or name.removeprefix(f"{family_root}/") in wildcard_models for name in model_names ) def unregistered_providers(rows: Mapping[str, object]) -> list[str]: + labelled_rows: Final = tuple( + (name, entry["litellm_provider"]) + for name, entry in rows.items() + if name != "sample_spec" + and isinstance(entry, dict) + and "litellm_provider" in entry + and entry.get("mode") not in MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY + and entry["litellm_provider"] not in PROVIDER_LABELS_WITHOUT_A_MODEL_SET + and entry["litellm_provider"] not in VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST + ) return sorted( - { - entry["litellm_provider"] - for name, entry in rows.items() - if name != "sample_spec" - and isinstance(entry, dict) - and "litellm_provider" in entry - and entry.get("mode") not in MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY - and entry["litellm_provider"] not in PROVIDER_LABELS_WITHOUT_A_MODEL_SET - and not is_registered_provider(entry["litellm_provider"]) - } + label + for label in {label for _, label in labelled_rows} + if not is_registered_provider(label, tuple(name for name, row_label in labelled_rows if row_label == label)) ) @pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) def test_every_cost_map_provider_is_registered(path: Path): assert unregistered_providers(json.loads(path.read_text())) == [], ( - f"{path.name} carries a litellm_provider that litellm.models_by_provider does not know, so a `/*` " - "grant expands to no models. Add a `_models` set in litellm/__init__.py, fill it in " - "_populate_provider_model_sets, and list it in _build_models_by_provider" + f"{path.name} carries a litellm_provider whose models a `/*` grant does not list. A new provider " + "needs a `_models` set in litellm/__init__.py, filled in _populate_provider_model_sets and listed " + "in _build_models_by_provider. A new `-` label needs its rows added to a set that " + "`models_by_provider[]` includes" ) def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): + wired_vertex_model: Final = sorted(litellm.vertex_language_models)[0] rows: Final = { "sample_spec": {"litellm_provider": "one of the supported providers", "mode": "chat"}, "nobody_registered/StartJob": {"litellm_provider": "nobody_registered", "mode": "audio_transcription"}, "gpt-4o": {"litellm_provider": "openai", "mode": "chat"}, + wired_vertex_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}, "vertex_ai/new-family-model": {"litellm_provider": "vertex_ai-new_family_models", "mode": "chat"}, "unknown_root/model": {"litellm_provider": "unknown_root-new_family_models", "mode": "chat"}, "some_search/search": {"litellm_provider": "some_search", "mode": "search"}, } - assert unregistered_providers(rows) == ["nobody_registered", "unknown_root-new_family_models"] + assert unregistered_providers(rows) == [ + "nobody_registered", + "unknown_root-new_family_models", + "vertex_ai-new_family_models", + ]