From 99636dff2b661a965689c998587c481d137c18c8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:50:24 +0000 Subject: [PATCH 1/6] fix(proxy): expand partial wildcard prefixes against provider-stripped model ids Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 23 ++- litellm/proxy/auth/model_checks.py | 51 +++--- .../test_wildcard_partial_prefix_expansion.py | 149 ++++++++++++++++++ .../proxy/auth/test_model_checks.py | 64 +++++--- .../test_health_check_helpers.py | 18 +++ 5 files changed, 260 insertions(+), 45 deletions(-) create mode 100644 tests/e2e/test_wildcard_partial_prefix_expansion.py diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index a0f027cd58f..16342728774 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,7 +3,7 @@ Helper functions for health check calls. """ import base64 -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from typing import TYPE_CHECKING, Final, Literal from litellm.llms.base_llm.ocr.transformation import DocumentType @@ -39,6 +39,19 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document return native(model, custom_llm_provider) +def _wildcard_health_check_models(wildcard_model: str, cheapest_models: Sequence[str]) -> tuple[str, ...]: + """Substitute each candidate id into the wildcard's dynamic segment so the health + check probes a name the wildcard route actually expands, instead of the bare + cost-map id (e.g. `databricks/system.ai.*` + `databricks/databricks-x` -> + `databricks/system.ai.databricks-x`).""" + provider_prefix, sep, wildcard_suffix = wildcard_model.partition("/") + if not sep or wildcard_suffix == "*": + return tuple(cheapest_models) + return tuple( + f"{provider_prefix}/{wildcard_suffix.replace('*', model.partition('/')[-1], 1)}" for model in cheapest_models + ) + + class HealthCheckHelpers: @staticmethod async def ahealth_check_wildcard_models( @@ -58,11 +71,9 @@ class HealthCheckHelpers: raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) - if len(cheapest_models) > 1: - fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list - else: - fallback_models = None - model_params["model"] = cheapest_models[0] + candidates: Final = _wildcard_health_check_models(wildcard_model=model, cheapest_models=cheapest_models) + fallback_models: Final = list(candidates[1:]) or None + model_params["model"] = candidates[0] model_params["litellm_logging_obj"] = litellm_logging_obj model_params["fallbacks"] = fallback_models model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16 diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index de2ca4762f1..944ed80280a 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -262,6 +262,24 @@ def _hydrate_litellm_credential_name( return litellm_params +def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> str: + leading, sep, model_suffix = model.partition("/") + return model_suffix if sep and leading in known_providers else model + + +def _apply_partial_wildcard_prefix( + provider_models: list[str], model_prefix: str, known_providers: frozenset[str] +) -> list[str]: + prefix_filtered: Final = [ + model + for model in provider_models + if _strip_known_provider_prefix(model, known_providers).startswith(model_prefix) + ] + if prefix_filtered: + return prefix_filtered + return [f"{model_prefix}{_strip_known_provider_prefix(model, known_providers)}" for model in provider_models] + + def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: wildcard_model_to_expand: Final = ( litellm_params.model @@ -288,23 +306,24 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ litellm_params = _hydrate_litellm_credential_name(litellm_params) - wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params) + provider_models: Final = get_provider_models(provider=provider, litellm_params=litellm_params) - if wildcard_models is None: + if provider_models is None: return [] - if wildcard_suffix != "*": - ## CHECK IF PARTIAL FILTER e.g. `gemini-*` - model_prefix: Final = wildcard_suffix.replace("*", "") - is_partial_filter: Final = any(wc_model.startswith(model_prefix) for wc_model in wildcard_models) - if is_partial_filter: - filtered_wildcard_models = [wc_model for wc_model in wildcard_models if wc_model.startswith(model_prefix)] - wildcard_models = filtered_wildcard_models - else: - # add model prefix to wildcard models - wildcard_models = [f"{model_prefix}{model}" for model in wildcard_models] + known_providers: Final = frozenset(provider.value for provider in LlmProviders) + + ## CHECK IF PARTIAL FILTER e.g. `gemini-*` + wildcard_models: Final = ( + provider_models + if wildcard_suffix == "*" + else _apply_partial_wildcard_prefix( + provider_models=provider_models, + model_prefix=wildcard_suffix.replace("*", ""), + known_providers=known_providers, + ) + ) - known_providers: Final = {provider.value for provider in LlmProviders} suffix_appended_wildcard_models: Final = [] for model in wildcard_models: if not model.startswith(wildcard_provider_prefix): @@ -314,11 +333,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b". # Only strip the leading segment when it is a known provider, so ids whose first # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. - leading, sep, model_suffix = model.partition("/") - if sep and leading in known_providers: - model = f"{wildcard_provider_prefix}/{model_suffix}" - else: - model = f"{wildcard_provider_prefix}/{model}" + model = f"{wildcard_provider_prefix}/{_strip_known_provider_prefix(model, known_providers)}" suffix_appended_wildcard_models.append(model) return suffix_appended_wildcard_models or [] diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py new file mode 100644 index 00000000000..48b265c2c62 --- /dev/null +++ b/tests/e2e/test_wildcard_partial_prefix_expansion.py @@ -0,0 +1,149 @@ +"""Regression: wildcard expansion of a partial provider prefix like +`databricks/system.ai.*` must produce `databricks/system.ai.` ids, +not `databricks/system.ai.databricks/databricks-`. + +On main, get_known_models_from_wildcard prepends the literal prefix before the +`*` ("system.ai.") to cost-map keys that already carry the `databricks/` +provider prefix, so /v1/models and /model/info list names that Databricks +rejects with "Invalid Unity Catalog name". See LIT-8910. + +The test boots its own proxy (the shared stack's model_list is fixed), with two +wildcard deployments: `databricks/system.ai.*` under test and `databricks/*` as +a control whose expansion must stay `databricks/databricks-*`. +""" + +import socket +import subprocess +import sys +import time +from collections.abc import Generator +from pathlib import Path +from typing import Final, cast + +import pytest +from e2e_http import URL, AuthHeaders, NoBody, get, probe, unwrap +from models import ModelsListParams, ModelsListResponse +from pydantic import BaseModel, ConfigDict + +REPO_ROOT: Final = Path(__file__).resolve().parents[2] + +MASTER_KEY: Final = "sk-1234" +AUTH: Final = AuthHeaders(authorization=f"Bearer {MASTER_KEY}") + +CONFIG_YAML: Final = """\ +model_list: + - model_name: databricks/system.ai.* + litellm_params: + model: databricks/system.ai.* + api_key: fake-databricks-token + api_base: https://example.invalid + - model_name: databricks/* + litellm_params: + model: databricks/* + api_key: fake-databricks-token + api_base: https://example.invalid +general_settings: + master_key: sk-1234 +""" + + +class _ModelInfoParams(BaseModel): + model_config = ConfigDict(protected_namespaces=(), extra="ignore") + model: str = "" + + +class _ModelInfoRow(BaseModel): + model_name: str + litellm_params: _ModelInfoParams = _ModelInfoParams() + + +class _ModelInfoResponse(BaseModel): + data: list[_ModelInfoRow] = [] + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return cast("tuple[str, int]", sock.getsockname())[1] + + +@pytest.fixture(scope="module") +def wildcard_proxy(tmp_path_factory: pytest.TempPathFactory) -> Generator[str]: + config_path = tmp_path_factory.mktemp("wildcard") / "config.yaml" + config_path.write_text(CONFIG_YAML) + port = _free_port() + proc = subprocess.Popen( + [ + sys.executable, + str(REPO_ROOT / "litellm/proxy/proxy_cli.py"), + "--config", + str(config_path), + "--port", + str(port), + ], + cwd=REPO_ROOT, + env={ + "PATH": "/usr/bin:/bin:/usr/local/bin", + "HOME": str(Path.home()), + "PYTHONPATH": str(REPO_ROOT), + "LITELLM_LOCAL_MODEL_COST_MAP": "true", + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true", + "LITELLM_MASTER_KEY": MASTER_KEY, + }, + ) + base = f"http://127.0.0.1:{port}" + try: + deadline = time.time() + 180 + while True: + alive = probe(URL(f"{base}/health/liveliness"), headers=NoBody(), params=NoBody(), timeout=5.0) + if alive.healthy: + break + if proc.poll() is not None: + pytest.fail(f"proxy exited during boot with {proc.returncode}") + assert time.time() < deadline, "proxy did not answer /health/liveliness within 180s" + time.sleep(2) + yield base + finally: + proc.terminate() + proc.wait(timeout=30) + + +def _list_model_ids(base: str) -> list[str]: + listed = get( + URL(f"{base}/v1/models"), + headers=AUTH, + params=ModelsListParams(return_wildcard_routes=False), + response_type=ModelsListResponse, + ) + return [entry.id for entry in unwrap(listed).data] + + +def test_partial_prefix_wildcard_expands_to_unity_catalog_names(wildcard_proxy: str) -> None: + ids = _list_model_ids(wildcard_proxy) + + system_ai_ids = [model_id for model_id in ids if model_id.startswith("databricks/system.ai.")] + assert system_ai_ids, f"no databricks/system.ai.* expansion in {ids}" + + malformed = [model_id for model_id in system_ai_ids if "/" in model_id.removeprefix("databricks/system.ai.")] + assert not malformed, f"expanded ids embed the provider-prefixed cost-map key after 'system.ai.': {malformed}" + + +def test_provider_wildcard_still_expands_to_cost_map_names(wildcard_proxy: str) -> None: + ids = _list_model_ids(wildcard_proxy) + + databricks_ids = [model_id for model_id in ids if model_id.startswith("databricks/databricks-")] + assert databricks_ids, f"databricks/* did not expand to cost-map names: {ids}" + + +def test_model_info_keeps_provider_model_for_expanded_deployments(wildcard_proxy: str) -> None: + info = unwrap( + get( + URL(f"{wildcard_proxy}/model/info"), + headers=AUTH, + params=NoBody(), + response_type=_ModelInfoResponse, + ) + ) + + bad = [row.model_name for row in info.data if "system.ai.databricks/" in row.litellm_params.model] + assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 13171a42cda..9204a69a89e 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -1,3 +1,4 @@ +from typing import Final from unittest.mock import patch import pytest @@ -11,9 +12,7 @@ def test_get_team_models_for_all_models_and_team_only_models(): model_access_groups = {} include_model_access_groups = False - result = get_team_models( - team_models, proxy_model_list, model_access_groups, include_model_access_groups - ) + result = get_team_models(team_models, proxy_model_list, model_access_groups, include_model_access_groups) combined_models = team_models + proxy_model_list assert set(result) == set(combined_models) @@ -246,9 +245,7 @@ def test_get_key_models_does_not_mutate_input(): ), ], ) -def test_get_complete_model_list_order( - key_models, team_models, proxy_model_list, model_list, expected -): +def test_get_complete_model_list_order(key_models, team_models, proxy_model_list, model_list, expected): """ Test that get_complete_model_list preserves order """ @@ -401,9 +398,7 @@ def test_wildcard_credential_hydration_preserves_deployment_params( captured_params["api_key"] = litellm_params.api_key captured_params["api_version"] = litellm_params.api_version captured_params["credential_name"] = litellm_params.litellm_credential_name - captured_params["has_unexpected_field"] = hasattr( - litellm_params, "unexpected_field" - ) + captured_params["has_unexpected_field"] = hasattr(litellm_params, "unexpected_field") return ["gpt-4o"] monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) @@ -448,9 +443,7 @@ def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): result = get_known_models_from_wildcard( wildcard_model="ollama_server1/*", - litellm_params=LiteLLM_Params( - model="ollama_chat/*", custom_llm_provider="ollama_chat" - ), + litellm_params=LiteLLM_Params(model="ollama_chat/*", custom_llm_provider="ollama_chat"), ) assert result == ["ollama_server1/gemma3:1b", "ollama_server1/llama3:8b"] @@ -477,14 +470,47 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment result = get_known_models_from_wildcard( wildcard_model="my_hf/*", - litellm_params=LiteLLM_Params( - model="huggingface/*", custom_llm_provider="huggingface" - ), + litellm_params=LiteLLM_Params(model="huggingface/*", custom_llm_provider="huggingface"), ) assert result == ["my_hf/meta-llama/Llama-3-8B"] +def test_partial_prefix_wildcard_expands_against_stripped_provider_ids(monkeypatch): + """A partial wildcard like ``databricks/system.ai.*`` must splice the prefix before + ``*`` onto the provider-stripped model id, not onto the whole cost-map key. Prepending + to the provider-prefixed key yielded ``databricks/system.ai.databricks/databricks-x``, + an id no upstream accepts. + """ + import litellm + from litellm.proxy.auth import model_checks + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + provider_ids: Final = list(litellm.models_by_provider["databricks"]) + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: provider_ids, + ) + + result: Final = get_known_models_from_wildcard( + wildcard_model="databricks/system.ai.*", + litellm_params=LiteLLM_Params( + model="databricks/system.ai.*", + api_key="x", + api_base="https://example.invalid", + ), + ) + + expected: Final = [f"databricks/system.ai.{model_id.partition('/')[-1]}" for model_id in provider_ids] + assert result == expected + assert all( + model_id.startswith("databricks/system.ai.") and "/" not in model_id.removeprefix("databricks/system.ai.") + for model_id in result + ) + + def test_wildcard_credential_hydration_preserves_missing_credential_name( monkeypatch, ): @@ -927,9 +953,7 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] try: litellm.add_known_models( - model_cost_map={ - fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"} - } + model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}} ) assert fake_model in litellm.models_by_provider["vertex_ai"] assert litellm.models_by_provider is captured_reference @@ -977,6 +1001,4 @@ def test_transcribe_is_a_known_provider_for_wildcard_expansion(): assert "transcribe" in litellm.models_by_provider assert "transcribe/StartTranscriptionJob" in litellm.models_by_provider["transcribe"] assert get_provider_models("transcribe") == ["transcribe/StartTranscriptionJob"] - assert get_known_models_from_wildcard("transcribe/*") == [ - "transcribe/StartTranscriptionJob" - ] + assert get_known_models_from_wildcard("transcribe/*") == ["transcribe/StartTranscriptionJob"] diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 47c4576f91f..4925bcb0b5d 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -14,6 +14,7 @@ from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.litellm_core_utils.health_check_helpers import ( IMAGE_EDIT_HEALTH_CHECK_PROMPT, HealthCheckHelpers, + _wildcard_health_check_models, ) from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth @@ -548,3 +549,20 @@ def test_ocr_health_check_document_raises_without_the_extension(): _ocr_health_check_document(model="mistral/mistral-ocr-latest", custom_llm_provider="mistral") finally: NATIVE_OCR_HEALTH_CHECK_DOCUMENT.reset() + + +def test_wildcard_health_check_models_partial_prefix_substitutes_stripped_id(): + candidates: Final = _wildcard_health_check_models( + "databricks/system.ai.*", + ["databricks/databricks-gemini-3-1-flash-image", "databricks/databricks-gpt-5"], + ) + assert candidates == ( + "databricks/system.ai.databricks-gemini-3-1-flash-image", + "databricks/system.ai.databricks-gpt-5", + ) + + +def test_wildcard_health_check_models_bare_provider_and_star_wildcard_unchanged(): + candidates: Final = ("databricks/databricks-gpt-5",) + assert _wildcard_health_check_models("databricks/*", candidates) == candidates + assert _wildcard_health_check_models("*", candidates) == candidates From 9398790ba23ff03a816d40f7bb959720ecc811ca Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:54:32 +0000 Subject: [PATCH 2/6] fix(health): splice wildcard prefix onto provider-stripped health check candidates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 19 +++++++---- .../proxy/auth/test_model_checks.py | 33 ++++++++++++------- .../test_health_check_helpers.py | 8 +++++ 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 16342728774..7ea9f9496d9 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Final, Literal from litellm.llms.base_llm.ocr.transformation import DocumentType from litellm.rust_bridge import runtime from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_HEALTH_CHECK_DOCUMENT -from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS +from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS, LlmProviders if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -39,16 +39,23 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document return native(model, custom_llm_provider) +def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> str: + leading, sep, model_suffix = model.partition("/") + return model_suffix if sep and leading in known_providers else model + + def _wildcard_health_check_models(wildcard_model: str, cheapest_models: Sequence[str]) -> tuple[str, ...]: - """Substitute each candidate id into the wildcard's dynamic segment so the health - check probes a name the wildcard route actually expands, instead of the bare - cost-map id (e.g. `databricks/system.ai.*` + `databricks/databricks-x` -> - `databricks/system.ai.databricks-x`).""" provider_prefix, sep, wildcard_suffix = wildcard_model.partition("/") if not sep or wildcard_suffix == "*": return tuple(cheapest_models) + known_providers: Final = frozenset(provider.value for provider in LlmProviders) + literal_prefix: Final = wildcard_suffix.replace("*", "") + stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in cheapest_models) return tuple( - f"{provider_prefix}/{wildcard_suffix.replace('*', model.partition('/')[-1], 1)}" for model in cheapest_models + f"{provider_prefix}/{stripped}" + if stripped.startswith(literal_prefix) + else f"{provider_prefix}/{wildcard_suffix.replace('*', stripped, 1)}" + for stripped in stripped_ids ) diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 9204a69a89e..866a8f42bdf 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -12,7 +12,9 @@ def test_get_team_models_for_all_models_and_team_only_models(): model_access_groups = {} include_model_access_groups = False - result = get_team_models(team_models, proxy_model_list, model_access_groups, include_model_access_groups) + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups + ) combined_models = team_models + proxy_model_list assert set(result) == set(combined_models) @@ -245,7 +247,9 @@ def test_get_key_models_does_not_mutate_input(): ), ], ) -def test_get_complete_model_list_order(key_models, team_models, proxy_model_list, model_list, expected): +def test_get_complete_model_list_order( + key_models, team_models, proxy_model_list, model_list, expected +): """ Test that get_complete_model_list preserves order """ @@ -398,7 +402,9 @@ def test_wildcard_credential_hydration_preserves_deployment_params( captured_params["api_key"] = litellm_params.api_key captured_params["api_version"] = litellm_params.api_version captured_params["credential_name"] = litellm_params.litellm_credential_name - captured_params["has_unexpected_field"] = hasattr(litellm_params, "unexpected_field") + captured_params["has_unexpected_field"] = hasattr( + litellm_params, "unexpected_field" + ) return ["gpt-4o"] monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) @@ -443,7 +449,9 @@ def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): result = get_known_models_from_wildcard( wildcard_model="ollama_server1/*", - litellm_params=LiteLLM_Params(model="ollama_chat/*", custom_llm_provider="ollama_chat"), + litellm_params=LiteLLM_Params( + model="ollama_chat/*", custom_llm_provider="ollama_chat" + ), ) assert result == ["ollama_server1/gemma3:1b", "ollama_server1/llama3:8b"] @@ -470,18 +478,15 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment result = get_known_models_from_wildcard( wildcard_model="my_hf/*", - litellm_params=LiteLLM_Params(model="huggingface/*", custom_llm_provider="huggingface"), + litellm_params=LiteLLM_Params( + model="huggingface/*", custom_llm_provider="huggingface" + ), ) assert result == ["my_hf/meta-llama/Llama-3-8B"] def test_partial_prefix_wildcard_expands_against_stripped_provider_ids(monkeypatch): - """A partial wildcard like ``databricks/system.ai.*`` must splice the prefix before - ``*`` onto the provider-stripped model id, not onto the whole cost-map key. Prepending - to the provider-prefixed key yielded ``databricks/system.ai.databricks/databricks-x``, - an id no upstream accepts. - """ import litellm from litellm.proxy.auth import model_checks from litellm.proxy.auth.model_checks import get_known_models_from_wildcard @@ -953,7 +958,9 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] try: litellm.add_known_models( - model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}} + model_cost_map={ + fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"} + } ) assert fake_model in litellm.models_by_provider["vertex_ai"] assert litellm.models_by_provider is captured_reference @@ -1001,4 +1008,6 @@ def test_transcribe_is_a_known_provider_for_wildcard_expansion(): assert "transcribe" in litellm.models_by_provider assert "transcribe/StartTranscriptionJob" in litellm.models_by_provider["transcribe"] assert get_provider_models("transcribe") == ["transcribe/StartTranscriptionJob"] - assert get_known_models_from_wildcard("transcribe/*") == ["transcribe/StartTranscriptionJob"] + assert get_known_models_from_wildcard("transcribe/*") == [ + "transcribe/StartTranscriptionJob" + ] diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 4925bcb0b5d..2e010bdf293 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -566,3 +566,11 @@ def test_wildcard_health_check_models_bare_provider_and_star_wildcard_unchanged( candidates: Final = ("databricks/databricks-gpt-5",) assert _wildcard_health_check_models("databricks/*", candidates) == candidates assert _wildcard_health_check_models("*", candidates) == candidates + + +def test_wildcard_health_check_models_partial_prefix_matching_literal_keeps_stripped_id(): + assert _wildcard_health_check_models("openai/gpt-4*", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) + + +def test_wildcard_health_check_models_partial_prefix_splices_suffix_around_star(): + assert _wildcard_health_check_models("openai/ft:*", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) From b24ef8bde68cb1b4e56c7a9f8a58f1f208173b27 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:02:22 +0000 Subject: [PATCH 3/6] fix(health): use provider-stripped wildcard suffix at the ahealth_check call site Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 15 ++++---- .../test_health_check_helpers.py | 34 +++++++++++++++---- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 7ea9f9496d9..30e9b020245 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -44,17 +44,18 @@ def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> return model_suffix if sep and leading in known_providers else model -def _wildcard_health_check_models(wildcard_model: str, cheapest_models: Sequence[str]) -> tuple[str, ...]: - provider_prefix, sep, wildcard_suffix = wildcard_model.partition("/") - if not sep or wildcard_suffix == "*": +def _wildcard_health_check_models( + wildcard_suffix: str, custom_llm_provider: str, cheapest_models: Sequence[str] +) -> tuple[str, ...]: + if wildcard_suffix == "*": return tuple(cheapest_models) known_providers: Final = frozenset(provider.value for provider in LlmProviders) literal_prefix: Final = wildcard_suffix.replace("*", "") stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in cheapest_models) return tuple( - f"{provider_prefix}/{stripped}" + f"{custom_llm_provider}/{stripped}" if stripped.startswith(literal_prefix) - else f"{provider_prefix}/{wildcard_suffix.replace('*', stripped, 1)}" + else f"{custom_llm_provider}/{wildcard_suffix.replace('*', stripped, 1)}" for stripped in stripped_ids ) @@ -78,7 +79,9 @@ class HealthCheckHelpers: raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) - candidates: Final = _wildcard_health_check_models(wildcard_model=model, cheapest_models=cheapest_models) + candidates: Final = _wildcard_health_check_models( + wildcard_suffix=model, custom_llm_provider=custom_llm_provider, cheapest_models=cheapest_models + ) fallback_models: Final = list(candidates[1:]) or None model_params["model"] = candidates[0] model_params["litellm_logging_obj"] = litellm_logging_obj diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 2e010bdf293..1087d8be3ff 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -553,7 +553,8 @@ def test_ocr_health_check_document_raises_without_the_extension(): def test_wildcard_health_check_models_partial_prefix_substitutes_stripped_id(): candidates: Final = _wildcard_health_check_models( - "databricks/system.ai.*", + "system.ai.*", + "databricks", ["databricks/databricks-gemini-3-1-flash-image", "databricks/databricks-gpt-5"], ) assert candidates == ( @@ -562,15 +563,36 @@ def test_wildcard_health_check_models_partial_prefix_substitutes_stripped_id(): ) -def test_wildcard_health_check_models_bare_provider_and_star_wildcard_unchanged(): +def test_wildcard_health_check_models_star_wildcard_unchanged(): candidates: Final = ("databricks/databricks-gpt-5",) - assert _wildcard_health_check_models("databricks/*", candidates) == candidates - assert _wildcard_health_check_models("*", candidates) == candidates + assert _wildcard_health_check_models("*", "databricks", candidates) == candidates def test_wildcard_health_check_models_partial_prefix_matching_literal_keeps_stripped_id(): - assert _wildcard_health_check_models("openai/gpt-4*", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) + assert _wildcard_health_check_models("gpt-4*", "openai", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) def test_wildcard_health_check_models_partial_prefix_splices_suffix_around_star(): - assert _wildcard_health_check_models("openai/ft:*", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) + assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) + + +@pytest.mark.asyncio +async def test_ahealth_check_partial_prefix_wildcard_probes_spliced_model_id(): + from litellm.litellm_core_utils.llm_request_utils import pick_cheapest_chat_models_from_llm_provider + + cheapest: Final = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider="databricks", n=3) + stripped: Final = tuple(model_id.partition("/")[-1] for model_id in cheapest) + + captured: dict[str, object] = {} + + async def _capture(**kwargs): + captured.update(kwargs) + + with patch("litellm.acompletion", side_effect=_capture): + await ahealth_check( + model_params={"model": "databricks/system.ai.*", "api_key": "x", "api_base": "https://example.invalid"}, + mode="chat", + ) + + assert captured["model"] == f"databricks/system.ai.{stripped[0]}" + assert captured["fallbacks"] == [f"databricks/system.ai.{model_id}" for model_id in stripped[1:]] From 4248af21c961c7be54ebec6b3fd5ad9db2526efb Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:09:26 +0000 Subject: [PATCH 4/6] test(e2e): spawn wildcard repro proxy with isolated interpreter flag Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/test_wildcard_partial_prefix_expansion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py index 48b265c2c62..dbe85028e10 100644 --- a/tests/e2e/test_wildcard_partial_prefix_expansion.py +++ b/tests/e2e/test_wildcard_partial_prefix_expansion.py @@ -75,6 +75,7 @@ def wildcard_proxy(tmp_path_factory: pytest.TempPathFactory) -> Generator[str]: proc = subprocess.Popen( [ sys.executable, + "-P", str(REPO_ROOT / "litellm/proxy/proxy_cli.py"), "--config", str(config_path), From a5b18431f7779a5b934c0283177d5f9743de3d65 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 03:11:00 +0000 Subject: [PATCH 5/6] fix(health): filter wildcard health probes to ids matching the literal prefix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 20 +++++++++---------- .../test_wildcard_partial_prefix_expansion.py | 4 +++- .../test_health_check_helpers.py | 14 +++++++++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 30e9b020245..692435b3910 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -45,19 +45,17 @@ def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> def _wildcard_health_check_models( - wildcard_suffix: str, custom_llm_provider: str, cheapest_models: Sequence[str] + wildcard_suffix: str, custom_llm_provider: str, candidate_models: Sequence[str] ) -> tuple[str, ...]: if wildcard_suffix == "*": - return tuple(cheapest_models) + return tuple(candidate_models[:3]) known_providers: Final = frozenset(provider.value for provider in LlmProviders) literal_prefix: Final = wildcard_suffix.replace("*", "") - stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in cheapest_models) - return tuple( - f"{custom_llm_provider}/{stripped}" - if stripped.startswith(literal_prefix) - else f"{custom_llm_provider}/{wildcard_suffix.replace('*', stripped, 1)}" - for stripped in stripped_ids - ) + stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in candidate_models) + matching: Final = [stripped for stripped in stripped_ids if stripped.startswith(literal_prefix)] + if matching: + return tuple(f"{custom_llm_provider}/{stripped}" for stripped in matching[:3]) + return tuple(f"{custom_llm_provider}/{wildcard_suffix.replace('*', stripped, 1)}" for stripped in stripped_ids[:3]) class HealthCheckHelpers: @@ -74,13 +72,13 @@ class HealthCheckHelpers: ) # this is a wildcard model, we need to pick a random model from the provider - cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3) + cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=10_000) if len(cheapest_models) == 0: raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) candidates: Final = _wildcard_health_check_models( - wildcard_suffix=model, custom_llm_provider=custom_llm_provider, cheapest_models=cheapest_models + wildcard_suffix=model, custom_llm_provider=custom_llm_provider, candidate_models=cheapest_models ) fallback_models: Final = list(candidates[1:]) or None model_params["model"] = candidates[0] diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py index dbe85028e10..1281e9d921b 100644 --- a/tests/e2e/test_wildcard_partial_prefix_expansion.py +++ b/tests/e2e/test_wildcard_partial_prefix_expansion.py @@ -146,5 +146,7 @@ def test_model_info_keeps_provider_model_for_expanded_deployments(wildcard_proxy ) ) - bad = [row.model_name for row in info.data if "system.ai.databricks/" in row.litellm_params.model] + expanded = [row for row in info.data if row.model_name.startswith("databricks/system.ai.")] + assert expanded, "model/info returned no expanded rows for databricks/system.ai.*" + bad = [row.model_name for row in expanded if "system.ai.databricks/" in row.litellm_params.model] assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 1087d8be3ff..e5bd43369ec 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -564,14 +564,24 @@ def test_wildcard_health_check_models_partial_prefix_substitutes_stripped_id(): def test_wildcard_health_check_models_star_wildcard_unchanged(): - candidates: Final = ("databricks/databricks-gpt-5",) - assert _wildcard_health_check_models("*", "databricks", candidates) == candidates + candidates: Final = ("databricks/a", "databricks/b", "databricks/c", "databricks/d") + assert _wildcard_health_check_models("*", "databricks", candidates) == ( + "databricks/a", + "databricks/b", + "databricks/c", + ) def test_wildcard_health_check_models_partial_prefix_matching_literal_keeps_stripped_id(): assert _wildcard_health_check_models("gpt-4*", "openai", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) +def test_wildcard_health_check_models_partial_prefix_filters_candidates_to_literal_prefix(): + assert _wildcard_health_check_models( + "gpt-4*", "openai", ["gpt-5-nano", "gpt-4o-mini", "gpt-4.1-mini", "gpt-4o"] + ) == ("openai/gpt-4o-mini", "openai/gpt-4.1-mini", "openai/gpt-4o") + + def test_wildcard_health_check_models_partial_prefix_splices_suffix_around_star(): assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) From f5735591d7ba0b9df41e56bab8eeb1010a9d6188 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 03:18:48 +0000 Subject: [PATCH 6/6] test(integration): move partial wildcard expansion repro onto the shared proxy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_wildcard_partial_prefix_expansion.py | 152 ------------------ .../test_wildcard_partial_prefix_expansion.py | 71 ++++++++ 2 files changed, 71 insertions(+), 152 deletions(-) delete mode 100644 tests/e2e/test_wildcard_partial_prefix_expansion.py create mode 100644 tests/integration/routing/test_wildcard_partial_prefix_expansion.py diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py deleted file mode 100644 index 1281e9d921b..00000000000 --- a/tests/e2e/test_wildcard_partial_prefix_expansion.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Regression: wildcard expansion of a partial provider prefix like -`databricks/system.ai.*` must produce `databricks/system.ai.` ids, -not `databricks/system.ai.databricks/databricks-`. - -On main, get_known_models_from_wildcard prepends the literal prefix before the -`*` ("system.ai.") to cost-map keys that already carry the `databricks/` -provider prefix, so /v1/models and /model/info list names that Databricks -rejects with "Invalid Unity Catalog name". See LIT-8910. - -The test boots its own proxy (the shared stack's model_list is fixed), with two -wildcard deployments: `databricks/system.ai.*` under test and `databricks/*` as -a control whose expansion must stay `databricks/databricks-*`. -""" - -import socket -import subprocess -import sys -import time -from collections.abc import Generator -from pathlib import Path -from typing import Final, cast - -import pytest -from e2e_http import URL, AuthHeaders, NoBody, get, probe, unwrap -from models import ModelsListParams, ModelsListResponse -from pydantic import BaseModel, ConfigDict - -REPO_ROOT: Final = Path(__file__).resolve().parents[2] - -MASTER_KEY: Final = "sk-1234" -AUTH: Final = AuthHeaders(authorization=f"Bearer {MASTER_KEY}") - -CONFIG_YAML: Final = """\ -model_list: - - model_name: databricks/system.ai.* - litellm_params: - model: databricks/system.ai.* - api_key: fake-databricks-token - api_base: https://example.invalid - - model_name: databricks/* - litellm_params: - model: databricks/* - api_key: fake-databricks-token - api_base: https://example.invalid -general_settings: - master_key: sk-1234 -""" - - -class _ModelInfoParams(BaseModel): - model_config = ConfigDict(protected_namespaces=(), extra="ignore") - model: str = "" - - -class _ModelInfoRow(BaseModel): - model_name: str - litellm_params: _ModelInfoParams = _ModelInfoParams() - - -class _ModelInfoResponse(BaseModel): - data: list[_ModelInfoRow] = [] - - -def _free_port() -> int: - with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - return cast("tuple[str, int]", sock.getsockname())[1] - - -@pytest.fixture(scope="module") -def wildcard_proxy(tmp_path_factory: pytest.TempPathFactory) -> Generator[str]: - config_path = tmp_path_factory.mktemp("wildcard") / "config.yaml" - config_path.write_text(CONFIG_YAML) - port = _free_port() - proc = subprocess.Popen( - [ - sys.executable, - "-P", - str(REPO_ROOT / "litellm/proxy/proxy_cli.py"), - "--config", - str(config_path), - "--port", - str(port), - ], - cwd=REPO_ROOT, - env={ - "PATH": "/usr/bin:/bin:/usr/local/bin", - "HOME": str(Path.home()), - "PYTHONPATH": str(REPO_ROOT), - "LITELLM_LOCAL_MODEL_COST_MAP": "true", - "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true", - "LITELLM_MASTER_KEY": MASTER_KEY, - }, - ) - base = f"http://127.0.0.1:{port}" - try: - deadline = time.time() + 180 - while True: - alive = probe(URL(f"{base}/health/liveliness"), headers=NoBody(), params=NoBody(), timeout=5.0) - if alive.healthy: - break - if proc.poll() is not None: - pytest.fail(f"proxy exited during boot with {proc.returncode}") - assert time.time() < deadline, "proxy did not answer /health/liveliness within 180s" - time.sleep(2) - yield base - finally: - proc.terminate() - proc.wait(timeout=30) - - -def _list_model_ids(base: str) -> list[str]: - listed = get( - URL(f"{base}/v1/models"), - headers=AUTH, - params=ModelsListParams(return_wildcard_routes=False), - response_type=ModelsListResponse, - ) - return [entry.id for entry in unwrap(listed).data] - - -def test_partial_prefix_wildcard_expands_to_unity_catalog_names(wildcard_proxy: str) -> None: - ids = _list_model_ids(wildcard_proxy) - - system_ai_ids = [model_id for model_id in ids if model_id.startswith("databricks/system.ai.")] - assert system_ai_ids, f"no databricks/system.ai.* expansion in {ids}" - - malformed = [model_id for model_id in system_ai_ids if "/" in model_id.removeprefix("databricks/system.ai.")] - assert not malformed, f"expanded ids embed the provider-prefixed cost-map key after 'system.ai.': {malformed}" - - -def test_provider_wildcard_still_expands_to_cost_map_names(wildcard_proxy: str) -> None: - ids = _list_model_ids(wildcard_proxy) - - databricks_ids = [model_id for model_id in ids if model_id.startswith("databricks/databricks-")] - assert databricks_ids, f"databricks/* did not expand to cost-map names: {ids}" - - -def test_model_info_keeps_provider_model_for_expanded_deployments(wildcard_proxy: str) -> None: - info = unwrap( - get( - URL(f"{wildcard_proxy}/model/info"), - headers=AUTH, - params=NoBody(), - response_type=_ModelInfoResponse, - ) - ) - - expanded = [row for row in info.data if row.model_name.startswith("databricks/system.ai.")] - assert expanded, "model/info returned no expanded rows for databricks/system.ai.*" - bad = [row.model_name for row in expanded if "system.ai.databricks/" in row.litellm_params.model] - assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" diff --git a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py new file mode 100644 index 00000000000..01206a2163a --- /dev/null +++ b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py @@ -0,0 +1,71 @@ +"""Partial-prefix wildcards like ``databricks/system.ai.*`` must splice the literal +prefix onto provider-stripped cost-map ids, never onto the provider-prefixed key.""" + +from collections.abc import Iterator +from typing import Final + +import pytest + +from integration._support.client import Gateway, object_value, string_value + +PATTERNS: Final = ("databricks/system.ai.*", "databricks/*") + + +def _registered_model_id(gateway: Gateway, pattern: str) -> str: + created: Final = object_value( + gateway.post( + "/model/new", + { + "model_name": pattern, + "litellm_params": { + "model": pattern, + "api_key": "integration-wildcard-key", + "api_base": "https://example.invalid", + }, + }, + ) + ) + return string_value(object_value(created["model_info"])["id"]) + + +@pytest.fixture +def wildcard_deployments(gateway: Gateway) -> Iterator[None]: + with gateway.scenario() as scenario: + for pattern in PATTERNS: + scenario.cleanups.callback(scenario.delete_model, _registered_model_id(gateway, pattern)) + yield + + +def _listed_model_ids(gateway: Gateway) -> tuple[str, ...]: + data: Final = gateway.get("/v1/models")["data"] + assert isinstance(data, list) + return tuple(string_value(object_value(entry)["id"]) for entry in data) + + +def test_partial_prefix_wildcard_expands_to_unity_catalog_names(gateway: Gateway, wildcard_deployments: None) -> None: + prefixed: Final = tuple( + model_id for model_id in _listed_model_ids(gateway) if model_id.startswith("databricks/system.ai.") + ) + assert prefixed + assert all("/" not in model_id.removeprefix("databricks/system.ai.") for model_id in prefixed) + + +def test_provider_wildcard_still_expands_to_cost_map_names(gateway: Gateway, wildcard_deployments: None) -> None: + assert any(model_id.startswith("databricks/databricks-") for model_id in _listed_model_ids(gateway)) + + +def test_model_info_keeps_provider_model_for_expanded_deployments(gateway: Gateway, wildcard_deployments: None) -> None: + data: Final = gateway.get("/model/info")["data"] + assert isinstance(data, list) + expanded: Final = [ + row + for row in (object_value(entry) for entry in data) + if string_value(row["model_name"]).startswith("databricks/system.ai.") + ] + assert expanded, "model/info returned no expanded rows for databricks/system.ai.*" + bad: Final = [ + row["model_name"] + for row in expanded + if "system.ai.databricks/" in string_value(object_value(row["litellm_params"])["model"]) + ] + assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}"