From 812bbee0b31db5908274448261209de2f7585739 Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 17:00:41 +0000 Subject: [PATCH] fix(model_info): scope fill_missing backfill to the rule's providers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 66 ++++++++++++++----- ...odel_prices_and_context_window_backup.json | 8 +-- litellm/utils.py | 4 +- model_prices_and_context_window.json | 8 +-- .../test_fallback_generalizations.py | 62 +++++++++++++---- 5 files changed, 107 insertions(+), 41 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 816a6c10bf1..da75faa4136 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -34,10 +34,10 @@ rules never mix the two and never use ``extends``. A rule whose Rules are only consulted after exact and case-insensitive lookups miss, so an exact cost-map entry always takes precedence over any rule. -Rules flagged with ``fill_missing_fields: true`` also fill only keys missing -from an exact cost-map entry, while values already present on the entry win on -conflict. Only flagged capability rules participate in this fill; routing rules -never do. +Rules flagged with ``fill_missing_for_providers: [..]`` also fill only keys +missing from an exact cost-map entry when the entry's ``litellm_provider`` is +listed, while values already present on the entry win on conflict. Only flagged +capability rules participate in this fill; routing rules never do. Patterns are matched case-insensitively with ``re.search`` and are not implicitly anchored: a rule must include ``^`` and ``$`` to bind to the whole model name, @@ -62,7 +62,7 @@ PATTERN_FIELD: Final = "pattern" MODEL_INFO_FIELD: Final = "model_info" PROVIDER_KEY: Final = "litellm_provider" LEGACY_EXTENDS_FIELD: Final = "extends" -FILL_MISSING_FIELDS_FIELD: Final = "fill_missing_fields" +FILL_MISSING_FOR_PROVIDERS_FIELD: Final = "fill_missing_for_providers" def _resolve_legacy_extends(rules: list) -> list: @@ -104,7 +104,7 @@ class _RoutingRule: class _CapabilityRule: pattern: re.Pattern model_info: dict - fill_missing_fields: bool + fill_missing_for_providers: frozenset[str] _CompiledRule = _RoutingRule | _CapabilityRule @@ -132,9 +132,29 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: e, ) return () - fill_missing_fields: Final = rule.get(FILL_MISSING_FIELDS_FIELD) is True + if FILL_MISSING_FOR_PROVIDERS_FIELD not in rule: + fill_missing_for_providers: Final[frozenset[str]] = frozenset() + else: + raw_fill_missing_for_providers: Final = rule.get(FILL_MISSING_FOR_PROVIDERS_FIELD) + if not isinstance(raw_fill_missing_for_providers, (list, tuple)) or not all( + isinstance(provider, str) for provider in raw_fill_missing_for_providers + ): + verbose_logger.warning( + "LiteLLM: skipping malformed fallback generalization rule %s " + "('%s' must be a list of provider strings).", + rule.get(NAME_FIELD, pattern), + FILL_MISSING_FOR_PROVIDERS_FIELD, + ) + return () + fill_missing_for_providers = frozenset(raw_fill_missing_for_providers) if PROVIDER_KEY not in model_info: - return (_CapabilityRule(pattern=compiled, model_info=model_info, fill_missing_fields=fill_missing_fields),) + return ( + _CapabilityRule( + pattern=compiled, + model_info=model_info, + fill_missing_for_providers=fill_missing_for_providers, + ), + ) provider: Final = model_info[PROVIDER_KEY] if not isinstance(provider, str): verbose_logger.warning( @@ -148,7 +168,11 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: return (_RoutingRule(pattern=compiled, provider=provider),) return ( _RoutingRule(pattern=compiled, provider=provider), - _CapabilityRule(pattern=compiled, model_info=model_info, fill_missing_fields=fill_missing_fields), + _CapabilityRule( + pattern=compiled, + model_info=model_info, + fill_missing_for_providers=fill_missing_for_providers, + ), ) @@ -167,7 +191,7 @@ class _FallbackGeneralizations: self.rules = installed self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule)) self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule)) - self.fill_missing_rules = tuple(rule for rule in self.capability_rules if rule.fill_missing_fields) + self.fill_missing_rules = tuple(rule for rule in self.capability_rules if rule.fill_missing_for_providers) def match_routing(self, model: str) -> str | None: if not model: @@ -185,10 +209,14 @@ class _FallbackGeneralizations: return None return {key: value for model_info in matched for key, value in model_info.items()} - def match_fill_missing(self, model: str) -> dict | None: - if not model: + def match_fill_missing(self, model: str, provider: str) -> dict | None: + if not model or not provider: return None - matched = tuple(rule.model_info for rule in self.fill_missing_rules if rule.pattern.search(model) is not None) + matched = tuple( + rule.model_info + for rule in self.fill_missing_rules + if provider in rule.fill_missing_for_providers and rule.pattern.search(model) is not None + ) if not matched: return None fill_missing: Final = { @@ -233,10 +261,12 @@ def match_capability_generalizations(model: str) -> dict | None: return _registry.match_capabilities(model) -def match_fill_missing_generalizations(model: str) -> dict | None: - """Return the union of flagged capability rules matching ``model``. +def match_fill_missing_generalizations(model: str, provider: str) -> dict | None: + """Return flagged capability rules matching ``model`` for ``provider``. - Later rules override earlier ones on key conflicts. Returns ``None`` when no - flagged rule matches. O(number of rules); only call once exact lookups have matched. + Later rules override earlier ones on key conflicts. Only rules listing + ``provider`` in ``fill_missing_for_providers`` contribute. Returns ``None`` + when no flagged rule matches. O(number of rules); only call once exact + lookups have matched. """ - return _registry.match_fill_missing(model) + return _registry.match_fill_missing(model, provider) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 44f52248b97..c9e85c28420 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_fields": true, + "fill_missing_for_providers": ["openai"], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/litellm/utils.py b/litellm/utils.py index 70fea3db4ec..6af66010bd7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5822,7 +5822,9 @@ def _get_model_info_helper( _model_info = None if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - fill_missing: Final = match_fill_missing_generalizations(key) + fill_missing: Final = match_fill_missing_generalizations( + key, _model_info.get("litellm_provider", "") + ) if fill_missing is not None: _model_info = { **{k: v for k, v in fill_missing.items() if k not in _model_info}, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 44f52248b97..c9e85c28420 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_fields": true, + "fill_missing_for_providers": ["openai"], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index eaecbd0b87e..09870faaaf7 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -123,12 +123,13 @@ def test_fill_missing_requires_per_rule_opt_in(restore_generalizations): { "name": "opt-in", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"supports_vision": True}, }, ] ) - assert match_fill_missing_generalizations("acme-1") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1", "openai") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1", "azure") is None assert match_capability_generalizations("acme-1") == { "supports_reasoning": True, "supports_vision": True, @@ -137,31 +138,43 @@ def test_fill_missing_requires_per_rule_opt_in(restore_generalizations): restore_generalizations( [{"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}] ) - assert match_fill_missing_generalizations("acme-1") is None + assert match_fill_missing_generalizations("acme-1", "openai") is None restore_generalizations( [ { "name": "mixed", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"litellm_provider": "openai", "supports_vision": True}, } ] ) - assert match_fill_missing_generalizations("acme-1") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1", "openai") == {"supports_vision": True} restore_generalizations( [ { "name": "route", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"litellm_provider": "openai"}, } ] ) - assert match_fill_missing_generalizations("acme-1") is None + assert match_fill_missing_generalizations("acme-1", "openai") is None + + restore_generalizations( + [ + { + "name": "malformed", + "pattern": r"^acme-", + "fill_missing_for_providers": "openai", + "model_info": {"supports_vision": True}, + } + ] + ) + assert match_fill_missing_generalizations("acme-1", "openai") is None def test_routing_rules_are_excluded_from_capability_results(restore_generalizations): @@ -373,6 +386,12 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "litellm_provider": "openai", "mode": "image_generation", }, + "acme-other": { + "input_cost_per_token": 7e-6, + "output_cost_per_token": 8e-6, + "litellm_provider": "openrouter", + "mode": "chat", + }, }, ) restore_generalizations( @@ -380,7 +399,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp { "name": "acme-backfill", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"supports_reasoning": True, "max_tokens": 5}, } ] @@ -397,6 +416,9 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp assert bare["input_cost_per_token"] == 3e-6 assert bare["key"] == "acme-bare" + other = litellm.get_model_info("acme-other", custom_llm_provider="openrouter") + assert other.get("supports_reasoning") is None + image = litellm.get_model_info("acme-image", custom_llm_provider="openai") assert image.get("supports_reasoning") is None @@ -727,7 +749,7 @@ def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_m def test_shipped_wandb_rule_does_not_fill_missing_mapped_entries(shipped_cost_map): - assert match_fill_missing_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct") is None + assert match_fill_missing_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct", "wandb") is None def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): @@ -854,18 +876,25 @@ def test_shipped_openai_reasoning_rule_loses_to_mapped_entries(shipped_cost_map) [ ("azure/us/o1-2024-12-17", "azure"), ("github_copilot/gpt-5", "github_copilot"), + ("openrouter/openai/o1", "openrouter"), + ("perplexity/openai/gpt-5.4-mini", "perplexity"), ], ) -def test_shipped_openai_reasoning_rule_backfills_mapped_entries(shipped_cost_map, model, provider): +def test_shipped_openai_reasoning_rule_does_not_backfill_other_providers(shipped_cost_map, model, provider): assert model in litellm.model_cost raw_entry = litellm.model_cost[model] assert "supports_reasoning" not in raw_entry model_without_provider = model.removeprefix(f"{provider}/") - assert litellm.supports_reasoning(model=model_without_provider, custom_llm_provider=provider) is True info = litellm.get_model_info(model=model_without_provider, custom_llm_provider=provider) + assert info.get("supports_reasoning") is None assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0) +def test_shipped_openai_reasoning_rule_matches_only_openai(shipped_cost_map): + assert match_fill_missing_generalizations("gpt-5.4", "openai") == {"supports_reasoning": True} + assert match_fill_missing_generalizations("gpt-5.4", "openrouter") is None + + def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map): model = "gemini/deep-research-pro-preview-12-2025" assert model in litellm.model_cost @@ -877,7 +906,7 @@ def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map): assert info.get("supports_reasoning") is None -def test_shipped_claude_thinking_rules_backfill_without_family_limits(shipped_cost_map): +def test_shipped_claude_thinking_rules_backfill_only_anthropic(shipped_cost_map): model = "perplexity/anthropic/claude-sonnet-4-6" assert model in litellm.model_cost raw_entry = litellm.model_cost[model] @@ -885,6 +914,11 @@ def test_shipped_claude_thinking_rules_backfill_without_family_limits(shipped_co assert "max_input_tokens" not in raw_entry info = litellm.get_model_info(model="anthropic/claude-sonnet-4-6", custom_llm_provider="perplexity") - assert info["supports_adaptive_thinking"] is True - assert info["supports_legacy_thinking"] is True + assert info.get("supports_adaptive_thinking") is None + assert info.get("supports_legacy_thinking") is None assert info.get("max_input_tokens") is None + assert match_fill_missing_generalizations("claude-sonnet-4-6", "anthropic") == { + "supports_adaptive_thinking": True, + "supports_legacy_thinking": True, + } + assert match_fill_missing_generalizations("claude-sonnet-4-6", "perplexity") is None