From cee7215b24344a71ed9619b5911dcfa209a0f59e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 04:04:33 +0000 Subject: [PATCH 01/12] feat(model_info): opt-in field-level backfill from fallback generalization rules Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 31 ++++- ...odel_prices_and_context_window_backup.json | 4 + litellm/utils.py | 6 + model_prices_and_context_window.json | 4 + .../test_fallback_generalizations.py | 124 +++++++++++++++++- 5 files changed, 166 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 7739fc82c77..a5360309a70 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -34,6 +34,11 @@ rules never mix the two and never use ``extends``. A rule whose Rules are only consulted after exact and case-insensitive lookups miss, so an exact cost-map entry always takes precedence over any rule. +Rules flagged with ``backfill_exact_entries: true`` also fill only keys missing +from an exact cost-map entry, while values already present on the entry win on +conflict. Only flagged capability rules participate in this backfill; routing +rules never do. + Patterns are matched case-insensitively with ``re.search`` and are not implicitly anchored: a rule must include ``^`` and ``$`` to bind to the whole model name, otherwise it matches as a substring. Keeping anchoring in the regex makes the rule @@ -57,6 +62,7 @@ PATTERN_FIELD: Final = "pattern" MODEL_INFO_FIELD: Final = "model_info" PROVIDER_KEY: Final = "litellm_provider" LEGACY_EXTENDS_FIELD: Final = "extends" +BACKFILL_FIELD: Final = "backfill_exact_entries" def _resolve_legacy_extends(rules: list) -> list: @@ -98,6 +104,7 @@ class _RoutingRule: class _CapabilityRule: pattern: re.Pattern model_info: dict + backfill_exact_entries: bool _CompiledRule = _RoutingRule | _CapabilityRule @@ -125,8 +132,9 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: e, ) return () + backfill: Final = rule.get(BACKFILL_FIELD) is True if PROVIDER_KEY not in model_info: - return (_CapabilityRule(pattern=compiled, model_info=model_info),) + return (_CapabilityRule(pattern=compiled, model_info=model_info, backfill_exact_entries=backfill),) provider: Final = model_info[PROVIDER_KEY] if not isinstance(provider, str): verbose_logger.warning( @@ -140,7 +148,7 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: return (_RoutingRule(pattern=compiled, provider=provider),) return ( _RoutingRule(pattern=compiled, provider=provider), - _CapabilityRule(pattern=compiled, model_info=model_info), + _CapabilityRule(pattern=compiled, model_info=model_info, backfill_exact_entries=backfill), ) @@ -151,6 +159,7 @@ class _FallbackGeneralizations: self.rules: list = [] self.routing_rules: tuple = () self.capability_rules: tuple = () + self.backfill_rules: tuple = () def set_rules(self, rules: list | None) -> None: installed: Final = rules if isinstance(rules, list) else [] @@ -158,6 +167,7 @@ class _FallbackGeneralizations: self.rules = installed self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule)) self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule)) + self.backfill_rules = tuple(rule for rule in self.capability_rules if rule.backfill_exact_entries) def match_routing(self, model: str) -> str | None: if not model: @@ -175,6 +185,14 @@ class _FallbackGeneralizations: return None return {key: value for model_info in matched for key, value in model_info.items()} + def match_backfill(self, model: str) -> dict | None: + if not model: + return None + matched = tuple(rule.model_info for rule in self.backfill_rules if rule.pattern.search(model) is not None) + if not matched: + return None + return {key: value for model_info in matched for key, value in model_info.items()} + _registry: Final = _FallbackGeneralizations() @@ -210,3 +228,12 @@ def match_capability_generalizations(model: str) -> dict | None: capability rule matches. O(number of rules); only call once exact lookups have missed. """ return _registry.match_capabilities(model) + + +def match_backfill_generalizations(model: str) -> dict | None: + """Return the union of flagged capability rules matching ``model``. + + Later rules override earlier ones on key conflicts. Returns ``None`` when no + flagged rule matches. O(number of rules); only call once exact lookups have matched. + """ + return _registry.match_backfill(model) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 89d582db151..6ba3c240a94 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57717,6 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "backfill_exact_entries": true, "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57725,6 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", + "backfill_exact_entries": true, "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57741,6 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "backfill_exact_entries": true, "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57757,6 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", + "backfill_exact_entries": true, "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/litellm/utils.py b/litellm/utils.py index 04139a124b6..87f270fa757 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -83,6 +83,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.fallback_generalizations import ( + match_backfill_generalizations, match_capability_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -5819,6 +5820,11 @@ def _get_model_info_helper( ): _model_info = None + if _model_info is not None and key is not None: + backfill: Final = match_backfill_generalizations(key) + if backfill is not None: + _model_info = {**{k: v for k, v in backfill.items() if k not in _model_info}, **_model_info} + if _model_info is None: generalization: Final = _get_model_info_from_generalization( model=model, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 89d582db151..6ba3c240a94 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57717,6 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "backfill_exact_entries": true, "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57725,6 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", + "backfill_exact_entries": true, "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57741,6 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "backfill_exact_entries": true, "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57757,6 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", + "backfill_exact_entries": true, "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index b6e656f282a..0221b9bbffc 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -11,11 +11,11 @@ import logging import pytest - import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, + match_backfill_generalizations, match_capability_generalizations, match_routing_generalization, set_fallback_generalizations, @@ -116,6 +116,42 @@ def test_capability_union_is_last_wins_in_file_order(restore_generalizations): } +def test_backfill_requires_per_rule_opt_in(restore_generalizations): + restore_generalizations( + [ + {"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}, + { + "name": "opt-in", + "pattern": r"^acme-", + "backfill_exact_entries": True, + "model_info": {"supports_vision": True}, + }, + ] + ) + assert match_backfill_generalizations("acme-1") == {"supports_vision": True} + assert match_capability_generalizations("acme-1") == { + "supports_reasoning": True, + "supports_vision": True, + } + + restore_generalizations( + [{"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}] + ) + assert match_backfill_generalizations("acme-1") is None + + restore_generalizations( + [ + { + "name": "route", + "pattern": r"^acme-", + "backfill_exact_entries": True, + "model_info": {"litellm_provider": "openai"}, + } + ] + ) + assert match_backfill_generalizations("acme-1") is None + + def test_routing_rules_are_excluded_from_capability_results(restore_generalizations): restore_generalizations( [ @@ -299,6 +335,58 @@ def test_exact_entry_takes_precedence_over_rule(restore_generalizations): assert info["input_cost_per_token"] != 999.0 +def test_exact_entries_backfill_only_missing_fields(restore_generalizations, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + "acme-full": { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "litellm_provider": "openai", + "mode": "chat", + "max_tokens": 7, + "supports_reasoning": False, + }, + "acme-bare": { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 4e-6, + "litellm_provider": "openai", + "mode": "chat", + }, + }, + ) + restore_generalizations( + [ + { + "name": "acme-backfill", + "pattern": r"^acme-", + "backfill_exact_entries": True, + "model_info": {"supports_reasoning": True, "max_tokens": 5}, + } + ] + ) + litellm.get_model_info.cache_clear() + + full = litellm.get_model_info("acme-full", custom_llm_provider="openai") + assert full["supports_reasoning"] is False + assert full["max_tokens"] == 7 + + bare = litellm.get_model_info("acme-bare", custom_llm_provider="openai") + assert bare["supports_reasoning"] is True + assert bare["max_tokens"] == 5 + assert bare["input_cost_per_token"] == 3e-6 + assert bare["key"] == "acme-bare" + + restore_generalizations( + [{"name": "acme-backfill", "pattern": r"^acme-", "model_info": {"supports_reasoning": True, "max_tokens": 5}}] + ) + litellm.get_model_info.cache_clear() + unflagged = litellm.get_model_info("acme-bare", custom_llm_provider="openai") + assert unflagged.get("supports_reasoning") is None + + # --------------------------------------------------------------------------- # # Shipped rules (bundled cost map) # --------------------------------------------------------------------------- # @@ -605,6 +693,10 @@ def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_m assert litellm.supports_reasoning(model=model, custom_llm_provider="wandb") is False, model +def test_shipped_wandb_rule_does_not_backfill_mapped_entries(shipped_cost_map): + assert match_backfill_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct") is None + + def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): """``^wandb/`` is anchored, so it cannot leak onto another provider's ids.""" assert match_capability_generalizations("wandb/some-new-model") == {"supports_reasoning": True} @@ -722,3 +814,33 @@ def test_shipped_openai_reasoning_rule_skips_non_reasoning_gpt_ids(shipped_cost_ def test_shipped_openai_reasoning_rule_loses_to_mapped_entries(shipped_cost_map): assert "gpt-5-search-api" in litellm.model_cost assert litellm.supports_reasoning(model="gpt-5-search-api", custom_llm_provider="openai") is False + + +@pytest.mark.parametrize( + "model,provider", + [ + ("azure/us/o1-2024-12-17", "azure"), + ("github_copilot/gpt-5", "github_copilot"), + ], +) +def test_shipped_openai_reasoning_rule_backfills_mapped_entries(shipped_cost_map, model, provider): + assert model in litellm.model_cost + raw_entry = litellm.model_cost[model] + assert "supports_reasoning" not in raw_entry + model_without_provider = model.removeprefix(f"{provider}/") + assert litellm.supports_reasoning(model=model_without_provider, custom_llm_provider=provider) is True + info = litellm.get_model_info(model=model_without_provider, custom_llm_provider=provider) + assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0) + + +def test_shipped_claude_thinking_rules_backfill_without_family_limits(shipped_cost_map): + model = "perplexity/anthropic/claude-sonnet-4-6" + assert model in litellm.model_cost + raw_entry = litellm.model_cost[model] + assert "supports_adaptive_thinking" not in raw_entry + assert "max_input_tokens" not in raw_entry + + info = litellm.get_model_info(model="anthropic/claude-sonnet-4-6", custom_llm_provider="perplexity") + assert info["supports_adaptive_thinking"] is True + assert info["supports_legacy_thinking"] is True + assert info.get("max_input_tokens") is None From 42c708670d9977219ecbcec92bf2fc56cf1b377c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 04:22:41 +0000 Subject: [PATCH 02/12] fix(model_info): guard backfill by mode, drop provider key, tighten claude major regex Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 5 ++- ...odel_prices_and_context_window_backup.json | 8 ++-- litellm/utils.py | 3 +- model_prices_and_context_window.json | 8 ++-- .../test_fallback_generalizations.py | 44 +++++++++++++++++++ 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index a5360309a70..8a029f9a642 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -191,7 +191,10 @@ class _FallbackGeneralizations: matched = tuple(rule.model_info for rule in self.backfill_rules if rule.pattern.search(model) is not None) if not matched: return None - return {key: value for model_info in matched for key, value in model_info.items()} + backfill: Final = { + key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY + } + return backfill or None _registry: Final = _FallbackGeneralizations() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ba3c240a94..a06688faa5b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57716,9 +57716,9 @@ }, { "name": "claude-adaptive-thinking", - "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true } @@ -57742,9 +57742,9 @@ }, { "name": "claude-mid-conversation-system", - "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true } diff --git a/litellm/utils.py b/litellm/utils.py index 87f270fa757..74e7001effe 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -255,6 +255,7 @@ from litellm.types.utils import ( ) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} +_BACKFILL_MODES: Final = frozenset({"chat", "responses"}) # +-----------------------------------------------+ # | | @@ -5820,7 +5821,7 @@ def _get_model_info_helper( ): _model_info = None - if _model_info is not None and key is not None: + if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: backfill: Final = match_backfill_generalizations(key) if backfill is not None: _model_info = {**{k: v for k, v in backfill.items() if k not in _model_info}, **_model_info} diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6ba3c240a94..a06688faa5b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57716,9 +57716,9 @@ }, { "name": "claude-adaptive-thinking", - "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true } @@ -57742,9 +57742,9 @@ }, { "name": "claude-mid-conversation-system", - "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true } diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 0221b9bbffc..036abedd90c 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -139,6 +139,18 @@ def test_backfill_requires_per_rule_opt_in(restore_generalizations): ) assert match_backfill_generalizations("acme-1") is None + restore_generalizations( + [ + { + "name": "mixed", + "pattern": r"^acme-", + "backfill_exact_entries": True, + "model_info": {"litellm_provider": "openai", "supports_vision": True}, + } + ] + ) + assert match_backfill_generalizations("acme-1") == {"supports_vision": True} + restore_generalizations( [ { @@ -355,6 +367,12 @@ def test_exact_entries_backfill_only_missing_fields(restore_generalizations, mon "litellm_provider": "openai", "mode": "chat", }, + "acme-image": { + "input_cost_per_token": 5e-6, + "output_cost_per_token": 6e-6, + "litellm_provider": "openai", + "mode": "image_generation", + }, }, ) restore_generalizations( @@ -379,6 +397,9 @@ def test_exact_entries_backfill_only_missing_fields(restore_generalizations, mon assert bare["input_cost_per_token"] == 3e-6 assert bare["key"] == "acme-bare" + image = litellm.get_model_info("acme-image", custom_llm_provider="openai") + assert image.get("supports_reasoning") is None + restore_generalizations( [{"name": "acme-backfill", "pattern": r"^acme-", "model_info": {"supports_reasoning": True, "max_tokens": 5}}] ) @@ -503,6 +524,18 @@ def test_shipped_version_boundaries(shipped_cost_map, model, provider, adaptive, assert info.get("supports_mid_conversation_system") is mid_conversation, model +def test_shipped_claude_version_regex_excludes_undelimited_41(shipped_cost_map): + unmatched = match_capability_generalizations("github_copilot/claude-opus-41") + assert unmatched is None or "supports_adaptive_thinking" not in unmatched + assert unmatched is None or "supports_mid_conversation_system" not in unmatched + + for model in ("claude-opus-5", "claude-sonnet-4-8"): + matched = match_capability_generalizations(model) + assert matched is not None + assert matched["supports_adaptive_thinking"] is True + assert matched["supports_mid_conversation_system"] is True + + def test_shipped_rules_cover_new_families_like_fable_at_5_plus(shipped_cost_map): """Both version gates accept any claude-- id at major 5 or higher, bare major or major-minor, so a new family shaped like claude-fable-5 gets adaptive @@ -833,6 +866,17 @@ def test_shipped_openai_reasoning_rule_backfills_mapped_entries(shipped_cost_map assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0) +def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map): + model = "gemini/deep-research-pro-preview-12-2025" + assert model in litellm.model_cost + raw_entry = litellm.model_cost[model] + assert "supports_reasoning" not in raw_entry + assert raw_entry["mode"] == "image_generation" + + info = litellm.get_model_info("deep-research-pro-preview-12-2025", custom_llm_provider="gemini") + assert info.get("supports_reasoning") is None + + def test_shipped_claude_thinking_rules_backfill_without_family_limits(shipped_cost_map): model = "perplexity/anthropic/claude-sonnet-4-6" assert model in litellm.model_cost From fb2057fde7871c657c929d07135cfe7d343c7325 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 04:27:36 +0000 Subject: [PATCH 03/12] refactor(model_info): rename backfill_exact_entries to fill_missing_fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 32 +++++++++---------- ...odel_prices_and_context_window_backup.json | 8 ++--- litellm/utils.py | 11 ++++--- model_prices_and_context_window.json | 8 ++--- .../test_fallback_generalizations.py | 26 +++++++-------- 5 files changed, 44 insertions(+), 41 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 8a029f9a642..816a6c10bf1 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -34,10 +34,10 @@ rules never mix the two and never use ``extends``. A rule whose Rules are only consulted after exact and case-insensitive lookups miss, so an exact cost-map entry always takes precedence over any rule. -Rules flagged with ``backfill_exact_entries: true`` also fill only keys missing +Rules flagged with ``fill_missing_fields: true`` also fill only keys missing from an exact cost-map entry, while values already present on the entry win on -conflict. Only flagged capability rules participate in this backfill; routing -rules never do. +conflict. Only flagged capability rules participate in this fill; routing rules +never do. Patterns are matched case-insensitively with ``re.search`` and are not implicitly anchored: a rule must include ``^`` and ``$`` to bind to the whole model name, @@ -62,7 +62,7 @@ PATTERN_FIELD: Final = "pattern" MODEL_INFO_FIELD: Final = "model_info" PROVIDER_KEY: Final = "litellm_provider" LEGACY_EXTENDS_FIELD: Final = "extends" -BACKFILL_FIELD: Final = "backfill_exact_entries" +FILL_MISSING_FIELDS_FIELD: Final = "fill_missing_fields" def _resolve_legacy_extends(rules: list) -> list: @@ -104,7 +104,7 @@ class _RoutingRule: class _CapabilityRule: pattern: re.Pattern model_info: dict - backfill_exact_entries: bool + fill_missing_fields: bool _CompiledRule = _RoutingRule | _CapabilityRule @@ -132,9 +132,9 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: e, ) return () - backfill: Final = rule.get(BACKFILL_FIELD) is True + fill_missing_fields: Final = rule.get(FILL_MISSING_FIELDS_FIELD) is True if PROVIDER_KEY not in model_info: - return (_CapabilityRule(pattern=compiled, model_info=model_info, backfill_exact_entries=backfill),) + return (_CapabilityRule(pattern=compiled, model_info=model_info, fill_missing_fields=fill_missing_fields),) provider: Final = model_info[PROVIDER_KEY] if not isinstance(provider, str): verbose_logger.warning( @@ -148,7 +148,7 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: return (_RoutingRule(pattern=compiled, provider=provider),) return ( _RoutingRule(pattern=compiled, provider=provider), - _CapabilityRule(pattern=compiled, model_info=model_info, backfill_exact_entries=backfill), + _CapabilityRule(pattern=compiled, model_info=model_info, fill_missing_fields=fill_missing_fields), ) @@ -159,7 +159,7 @@ class _FallbackGeneralizations: self.rules: list = [] self.routing_rules: tuple = () self.capability_rules: tuple = () - self.backfill_rules: tuple = () + self.fill_missing_rules: tuple = () def set_rules(self, rules: list | None) -> None: installed: Final = rules if isinstance(rules, list) else [] @@ -167,7 +167,7 @@ class _FallbackGeneralizations: self.rules = installed self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule)) self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule)) - self.backfill_rules = tuple(rule for rule in self.capability_rules if rule.backfill_exact_entries) + self.fill_missing_rules = tuple(rule for rule in self.capability_rules if rule.fill_missing_fields) def match_routing(self, model: str) -> str | None: if not model: @@ -185,16 +185,16 @@ class _FallbackGeneralizations: return None return {key: value for model_info in matched for key, value in model_info.items()} - def match_backfill(self, model: str) -> dict | None: + def match_fill_missing(self, model: str) -> dict | None: if not model: return None - matched = tuple(rule.model_info for rule in self.backfill_rules if rule.pattern.search(model) is not None) + matched = tuple(rule.model_info for rule in self.fill_missing_rules if rule.pattern.search(model) is not None) if not matched: return None - backfill: Final = { + fill_missing: Final = { key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY } - return backfill or None + return fill_missing or None _registry: Final = _FallbackGeneralizations() @@ -233,10 +233,10 @@ def match_capability_generalizations(model: str) -> dict | None: return _registry.match_capabilities(model) -def match_backfill_generalizations(model: str) -> dict | None: +def match_fill_missing_generalizations(model: str) -> dict | None: """Return the union of flagged capability rules matching ``model``. Later rules override earlier ones on key conflicts. Returns ``None`` when no flagged rule matches. O(number of rules); only call once exact lookups have matched. """ - return _registry.match_backfill(model) + return _registry.match_fill_missing(model) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a06688faa5b..44f52248b97 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/litellm/utils.py b/litellm/utils.py index 74e7001effe..70fea3db4ec 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -83,8 +83,8 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.fallback_generalizations import ( - match_backfill_generalizations, match_capability_generalizations, + match_fill_missing_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -5822,9 +5822,12 @@ def _get_model_info_helper( _model_info = None if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - backfill: Final = match_backfill_generalizations(key) - if backfill is not None: - _model_info = {**{k: v for k, v in backfill.items() if k not in _model_info}, **_model_info} + fill_missing: Final = match_fill_missing_generalizations(key) + if fill_missing is not None: + _model_info = { + **{k: v for k, v in fill_missing.items() if k not in _model_info}, + **_model_info, + } if _model_info is None: generalization: Final = _get_model_info_from_generalization( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a06688faa5b..44f52248b97 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "backfill_exact_entries": true, + "fill_missing_fields": true, "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 036abedd90c..eaecbd0b87e 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -15,8 +15,8 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, - match_backfill_generalizations, match_capability_generalizations, + match_fill_missing_generalizations, match_routing_generalization, set_fallback_generalizations, ) @@ -116,19 +116,19 @@ def test_capability_union_is_last_wins_in_file_order(restore_generalizations): } -def test_backfill_requires_per_rule_opt_in(restore_generalizations): +def test_fill_missing_requires_per_rule_opt_in(restore_generalizations): restore_generalizations( [ {"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}, { "name": "opt-in", "pattern": r"^acme-", - "backfill_exact_entries": True, + "fill_missing_fields": True, "model_info": {"supports_vision": True}, }, ] ) - assert match_backfill_generalizations("acme-1") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1") == {"supports_vision": True} assert match_capability_generalizations("acme-1") == { "supports_reasoning": True, "supports_vision": True, @@ -137,31 +137,31 @@ def test_backfill_requires_per_rule_opt_in(restore_generalizations): restore_generalizations( [{"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}] ) - assert match_backfill_generalizations("acme-1") is None + assert match_fill_missing_generalizations("acme-1") is None restore_generalizations( [ { "name": "mixed", "pattern": r"^acme-", - "backfill_exact_entries": True, + "fill_missing_fields": True, "model_info": {"litellm_provider": "openai", "supports_vision": True}, } ] ) - assert match_backfill_generalizations("acme-1") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1") == {"supports_vision": True} restore_generalizations( [ { "name": "route", "pattern": r"^acme-", - "backfill_exact_entries": True, + "fill_missing_fields": True, "model_info": {"litellm_provider": "openai"}, } ] ) - assert match_backfill_generalizations("acme-1") is None + assert match_fill_missing_generalizations("acme-1") is None def test_routing_rules_are_excluded_from_capability_results(restore_generalizations): @@ -347,7 +347,7 @@ def test_exact_entry_takes_precedence_over_rule(restore_generalizations): assert info["input_cost_per_token"] != 999.0 -def test_exact_entries_backfill_only_missing_fields(restore_generalizations, monkeypatch): +def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeypatch): monkeypatch.setattr( litellm, "model_cost", @@ -380,7 +380,7 @@ def test_exact_entries_backfill_only_missing_fields(restore_generalizations, mon { "name": "acme-backfill", "pattern": r"^acme-", - "backfill_exact_entries": True, + "fill_missing_fields": True, "model_info": {"supports_reasoning": True, "max_tokens": 5}, } ] @@ -726,8 +726,8 @@ def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_m assert litellm.supports_reasoning(model=model, custom_llm_provider="wandb") is False, model -def test_shipped_wandb_rule_does_not_backfill_mapped_entries(shipped_cost_map): - assert match_backfill_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct") is None +def test_shipped_wandb_rule_does_not_fill_missing_mapped_entries(shipped_cost_map): + assert match_fill_missing_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct") is None def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): From 812bbee0b31db5908274448261209de2f7585739 Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 17:00:41 +0000 Subject: [PATCH 04/12] fix(model_info): scope fill_missing backfill to the rule's providers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 66 ++++++++++++++----- ...odel_prices_and_context_window_backup.json | 8 +-- litellm/utils.py | 4 +- model_prices_and_context_window.json | 8 +-- .../test_fallback_generalizations.py | 62 +++++++++++++---- 5 files changed, 107 insertions(+), 41 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 816a6c10bf1..da75faa4136 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -34,10 +34,10 @@ rules never mix the two and never use ``extends``. A rule whose Rules are only consulted after exact and case-insensitive lookups miss, so an exact cost-map entry always takes precedence over any rule. -Rules flagged with ``fill_missing_fields: true`` also fill only keys missing -from an exact cost-map entry, while values already present on the entry win on -conflict. Only flagged capability rules participate in this fill; routing rules -never do. +Rules flagged with ``fill_missing_for_providers: [..]`` also fill only keys +missing from an exact cost-map entry when the entry's ``litellm_provider`` is +listed, while values already present on the entry win on conflict. Only flagged +capability rules participate in this fill; routing rules never do. Patterns are matched case-insensitively with ``re.search`` and are not implicitly anchored: a rule must include ``^`` and ``$`` to bind to the whole model name, @@ -62,7 +62,7 @@ PATTERN_FIELD: Final = "pattern" MODEL_INFO_FIELD: Final = "model_info" PROVIDER_KEY: Final = "litellm_provider" LEGACY_EXTENDS_FIELD: Final = "extends" -FILL_MISSING_FIELDS_FIELD: Final = "fill_missing_fields" +FILL_MISSING_FOR_PROVIDERS_FIELD: Final = "fill_missing_for_providers" def _resolve_legacy_extends(rules: list) -> list: @@ -104,7 +104,7 @@ class _RoutingRule: class _CapabilityRule: pattern: re.Pattern model_info: dict - fill_missing_fields: bool + fill_missing_for_providers: frozenset[str] _CompiledRule = _RoutingRule | _CapabilityRule @@ -132,9 +132,29 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: e, ) return () - fill_missing_fields: Final = rule.get(FILL_MISSING_FIELDS_FIELD) is True + if FILL_MISSING_FOR_PROVIDERS_FIELD not in rule: + fill_missing_for_providers: Final[frozenset[str]] = frozenset() + else: + raw_fill_missing_for_providers: Final = rule.get(FILL_MISSING_FOR_PROVIDERS_FIELD) + if not isinstance(raw_fill_missing_for_providers, (list, tuple)) or not all( + isinstance(provider, str) for provider in raw_fill_missing_for_providers + ): + verbose_logger.warning( + "LiteLLM: skipping malformed fallback generalization rule %s " + "('%s' must be a list of provider strings).", + rule.get(NAME_FIELD, pattern), + FILL_MISSING_FOR_PROVIDERS_FIELD, + ) + return () + fill_missing_for_providers = frozenset(raw_fill_missing_for_providers) if PROVIDER_KEY not in model_info: - return (_CapabilityRule(pattern=compiled, model_info=model_info, fill_missing_fields=fill_missing_fields),) + return ( + _CapabilityRule( + pattern=compiled, + model_info=model_info, + fill_missing_for_providers=fill_missing_for_providers, + ), + ) provider: Final = model_info[PROVIDER_KEY] if not isinstance(provider, str): verbose_logger.warning( @@ -148,7 +168,11 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: return (_RoutingRule(pattern=compiled, provider=provider),) return ( _RoutingRule(pattern=compiled, provider=provider), - _CapabilityRule(pattern=compiled, model_info=model_info, fill_missing_fields=fill_missing_fields), + _CapabilityRule( + pattern=compiled, + model_info=model_info, + fill_missing_for_providers=fill_missing_for_providers, + ), ) @@ -167,7 +191,7 @@ class _FallbackGeneralizations: self.rules = installed self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule)) self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule)) - self.fill_missing_rules = tuple(rule for rule in self.capability_rules if rule.fill_missing_fields) + self.fill_missing_rules = tuple(rule for rule in self.capability_rules if rule.fill_missing_for_providers) def match_routing(self, model: str) -> str | None: if not model: @@ -185,10 +209,14 @@ class _FallbackGeneralizations: return None return {key: value for model_info in matched for key, value in model_info.items()} - def match_fill_missing(self, model: str) -> dict | None: - if not model: + def match_fill_missing(self, model: str, provider: str) -> dict | None: + if not model or not provider: return None - matched = tuple(rule.model_info for rule in self.fill_missing_rules if rule.pattern.search(model) is not None) + matched = tuple( + rule.model_info + for rule in self.fill_missing_rules + if provider in rule.fill_missing_for_providers and rule.pattern.search(model) is not None + ) if not matched: return None fill_missing: Final = { @@ -233,10 +261,12 @@ def match_capability_generalizations(model: str) -> dict | None: return _registry.match_capabilities(model) -def match_fill_missing_generalizations(model: str) -> dict | None: - """Return the union of flagged capability rules matching ``model``. +def match_fill_missing_generalizations(model: str, provider: str) -> dict | None: + """Return flagged capability rules matching ``model`` for ``provider``. - Later rules override earlier ones on key conflicts. Returns ``None`` when no - flagged rule matches. O(number of rules); only call once exact lookups have matched. + Later rules override earlier ones on key conflicts. Only rules listing + ``provider`` in ``fill_missing_for_providers`` contribute. Returns ``None`` + when no flagged rule matches. O(number of rules); only call once exact + lookups have matched. """ - return _registry.match_fill_missing(model) + return _registry.match_fill_missing(model, provider) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 44f52248b97..c9e85c28420 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_fields": true, + "fill_missing_for_providers": ["openai"], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/litellm/utils.py b/litellm/utils.py index 70fea3db4ec..6af66010bd7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5822,7 +5822,9 @@ def _get_model_info_helper( _model_info = None if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - fill_missing: Final = match_fill_missing_generalizations(key) + fill_missing: Final = match_fill_missing_generalizations( + key, _model_info.get("litellm_provider", "") + ) if fill_missing is not None: _model_info = { **{k: v for k, v in fill_missing.items() if k not in _model_info}, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 44f52248b97..c9e85c28420 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_fields": true, + "fill_missing_for_providers": ["anthropic"], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_fields": true, + "fill_missing_for_providers": ["openai"], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index eaecbd0b87e..09870faaaf7 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -123,12 +123,13 @@ def test_fill_missing_requires_per_rule_opt_in(restore_generalizations): { "name": "opt-in", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"supports_vision": True}, }, ] ) - assert match_fill_missing_generalizations("acme-1") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1", "openai") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1", "azure") is None assert match_capability_generalizations("acme-1") == { "supports_reasoning": True, "supports_vision": True, @@ -137,31 +138,43 @@ def test_fill_missing_requires_per_rule_opt_in(restore_generalizations): restore_generalizations( [{"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}] ) - assert match_fill_missing_generalizations("acme-1") is None + assert match_fill_missing_generalizations("acme-1", "openai") is None restore_generalizations( [ { "name": "mixed", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"litellm_provider": "openai", "supports_vision": True}, } ] ) - assert match_fill_missing_generalizations("acme-1") == {"supports_vision": True} + assert match_fill_missing_generalizations("acme-1", "openai") == {"supports_vision": True} restore_generalizations( [ { "name": "route", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"litellm_provider": "openai"}, } ] ) - assert match_fill_missing_generalizations("acme-1") is None + assert match_fill_missing_generalizations("acme-1", "openai") is None + + restore_generalizations( + [ + { + "name": "malformed", + "pattern": r"^acme-", + "fill_missing_for_providers": "openai", + "model_info": {"supports_vision": True}, + } + ] + ) + assert match_fill_missing_generalizations("acme-1", "openai") is None def test_routing_rules_are_excluded_from_capability_results(restore_generalizations): @@ -373,6 +386,12 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "litellm_provider": "openai", "mode": "image_generation", }, + "acme-other": { + "input_cost_per_token": 7e-6, + "output_cost_per_token": 8e-6, + "litellm_provider": "openrouter", + "mode": "chat", + }, }, ) restore_generalizations( @@ -380,7 +399,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp { "name": "acme-backfill", "pattern": r"^acme-", - "fill_missing_fields": True, + "fill_missing_for_providers": ["openai"], "model_info": {"supports_reasoning": True, "max_tokens": 5}, } ] @@ -397,6 +416,9 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp assert bare["input_cost_per_token"] == 3e-6 assert bare["key"] == "acme-bare" + other = litellm.get_model_info("acme-other", custom_llm_provider="openrouter") + assert other.get("supports_reasoning") is None + image = litellm.get_model_info("acme-image", custom_llm_provider="openai") assert image.get("supports_reasoning") is None @@ -727,7 +749,7 @@ def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_m def test_shipped_wandb_rule_does_not_fill_missing_mapped_entries(shipped_cost_map): - assert match_fill_missing_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct") is None + assert match_fill_missing_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct", "wandb") is None def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): @@ -854,18 +876,25 @@ def test_shipped_openai_reasoning_rule_loses_to_mapped_entries(shipped_cost_map) [ ("azure/us/o1-2024-12-17", "azure"), ("github_copilot/gpt-5", "github_copilot"), + ("openrouter/openai/o1", "openrouter"), + ("perplexity/openai/gpt-5.4-mini", "perplexity"), ], ) -def test_shipped_openai_reasoning_rule_backfills_mapped_entries(shipped_cost_map, model, provider): +def test_shipped_openai_reasoning_rule_does_not_backfill_other_providers(shipped_cost_map, model, provider): assert model in litellm.model_cost raw_entry = litellm.model_cost[model] assert "supports_reasoning" not in raw_entry model_without_provider = model.removeprefix(f"{provider}/") - assert litellm.supports_reasoning(model=model_without_provider, custom_llm_provider=provider) is True info = litellm.get_model_info(model=model_without_provider, custom_llm_provider=provider) + assert info.get("supports_reasoning") is None assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0) +def test_shipped_openai_reasoning_rule_matches_only_openai(shipped_cost_map): + assert match_fill_missing_generalizations("gpt-5.4", "openai") == {"supports_reasoning": True} + assert match_fill_missing_generalizations("gpt-5.4", "openrouter") is None + + def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map): model = "gemini/deep-research-pro-preview-12-2025" assert model in litellm.model_cost @@ -877,7 +906,7 @@ def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map): assert info.get("supports_reasoning") is None -def test_shipped_claude_thinking_rules_backfill_without_family_limits(shipped_cost_map): +def test_shipped_claude_thinking_rules_backfill_only_anthropic(shipped_cost_map): model = "perplexity/anthropic/claude-sonnet-4-6" assert model in litellm.model_cost raw_entry = litellm.model_cost[model] @@ -885,6 +914,11 @@ def test_shipped_claude_thinking_rules_backfill_without_family_limits(shipped_co assert "max_input_tokens" not in raw_entry info = litellm.get_model_info(model="anthropic/claude-sonnet-4-6", custom_llm_provider="perplexity") - assert info["supports_adaptive_thinking"] is True - assert info["supports_legacy_thinking"] is True + assert info.get("supports_adaptive_thinking") is None + assert info.get("supports_legacy_thinking") is None assert info.get("max_input_tokens") is None + assert match_fill_missing_generalizations("claude-sonnet-4-6", "anthropic") == { + "supports_adaptive_thinking": True, + "supports_legacy_thinking": True, + } + assert match_fill_missing_generalizations("claude-sonnet-4-6", "perplexity") is None From 71744fb9d19beff93d60cd5d49525e09f3382b8a Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 17:04:29 +0000 Subject: [PATCH 05/12] style(model_info): format provider backfill call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 6af66010bd7..aeeec669d24 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5822,9 +5822,7 @@ def _get_model_info_helper( _model_info = None if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - fill_missing: Final = match_fill_missing_generalizations( - key, _model_info.get("litellm_provider", "") - ) + fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", "")) if fill_missing is not None: _model_info = { **{k: v for k, v in fill_missing.items() if k not in _model_info}, From 6f086f74b32ce1b2a36790f11902f110224c344c Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 17:10:54 +0000 Subject: [PATCH 06/12] style(model_info): parameterize fill_missing annotations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/fallback_generalizations.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index da75faa4136..7ea4f3589d8 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -183,7 +183,7 @@ class _FallbackGeneralizations: self.rules: list = [] self.routing_rules: tuple = () self.capability_rules: tuple = () - self.fill_missing_rules: tuple = () + self.fill_missing_rules: tuple[_CapabilityRule, ...] = () def set_rules(self, rules: list | None) -> None: installed: Final = rules if isinstance(rules, list) else [] @@ -209,7 +209,7 @@ class _FallbackGeneralizations: return None return {key: value for model_info in matched for key, value in model_info.items()} - def match_fill_missing(self, model: str, provider: str) -> dict | None: + def match_fill_missing(self, model: str, provider: str) -> dict[str, object] | None: if not model or not provider: return None matched = tuple( @@ -219,7 +219,7 @@ class _FallbackGeneralizations: ) if not matched: return None - fill_missing: Final = { + fill_missing: Final[dict[str, object]] = { key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY } return fill_missing or None @@ -261,7 +261,7 @@ def match_capability_generalizations(model: str) -> dict | None: return _registry.match_capabilities(model) -def match_fill_missing_generalizations(model: str, provider: str) -> dict | None: +def match_fill_missing_generalizations(model: str, provider: str) -> dict[str, object] | None: """Return flagged capability rules matching ``model`` for ``provider``. Later rules override earlier ones on key conflicts. Only rules listing From e4e6b5daeb4a9e7a0acf4236ef63476a402a57eb Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 17:27:19 +0000 Subject: [PATCH 07/12] fix(lint): allow provider backfill dict annotations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/fallback_generalizations.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 7ea4f3589d8..feea52aa0bc 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -209,7 +209,9 @@ class _FallbackGeneralizations: return None return {key: value for model_info in matched for key, value in model_info.items()} - def match_fill_missing(self, model: str, provider: str) -> dict[str, object] | None: + def match_fill_missing( + self, model: str, provider: str + ) -> dict[str, object] | None: # mutable-ok: preserve the existing dict return contract if not model or not provider: return None matched = tuple( @@ -219,7 +221,7 @@ class _FallbackGeneralizations: ) if not matched: return None - fill_missing: Final[dict[str, object]] = { + fill_missing: Final[dict[str, object]] = { # mutable-ok: preserve the existing dict merge input key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY } return fill_missing or None @@ -261,7 +263,9 @@ def match_capability_generalizations(model: str) -> dict | None: return _registry.match_capabilities(model) -def match_fill_missing_generalizations(model: str, provider: str) -> dict[str, object] | None: +def match_fill_missing_generalizations( + model: str, provider: str +) -> dict[str, object] | None: # mutable-ok: preserve the existing dict return contract """Return flagged capability rules matching ``model`` for ``provider``. Later rules override earlier ones on key conflicts. Only rules listing From 5781678477196b7d26f0d20682e02ce4b74b1fac Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 17:33:28 +0000 Subject: [PATCH 08/12] refactor(model_info): drop litellm._logging import from fallback_generalizations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/fallback_generalizations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index feea52aa0bc..9b5c369b56e 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -51,12 +51,12 @@ Rules are compiled and classified once, at install time. The match functions are O(number of rules); callers must only invoke them on a cache miss. """ +import logging import re from dataclasses import dataclass from typing import Final -from litellm._logging import verbose_logger - +verbose_logger: Final = logging.getLogger("LiteLLM") NAME_FIELD: Final = "name" PATTERN_FIELD: Final = "pattern" MODEL_INFO_FIELD: Final = "model_info" From e366d3bd302e82ad85c58190f7e23a88b8d56b6b Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 17:35:24 +0000 Subject: [PATCH 09/12] refactor(model_info): return Mapping from fill_missing matchers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 9b5c369b56e..25ea1bff613 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -53,7 +53,9 @@ O(number of rules); callers must only invoke them on a cache miss. import logging import re +from collections.abc import Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import Final verbose_logger: Final = logging.getLogger("LiteLLM") @@ -209,9 +211,7 @@ class _FallbackGeneralizations: return None return {key: value for model_info in matched for key, value in model_info.items()} - def match_fill_missing( - self, model: str, provider: str - ) -> dict[str, object] | None: # mutable-ok: preserve the existing dict return contract + def match_fill_missing(self, model: str, provider: str) -> Mapping[str, object] | None: if not model or not provider: return None matched = tuple( @@ -221,9 +221,9 @@ class _FallbackGeneralizations: ) if not matched: return None - fill_missing: Final[dict[str, object]] = { # mutable-ok: preserve the existing dict merge input - key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY - } + fill_missing: Final[Mapping[str, object]] = MappingProxyType( + {key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY} + ) return fill_missing or None @@ -263,9 +263,7 @@ def match_capability_generalizations(model: str) -> dict | None: return _registry.match_capabilities(model) -def match_fill_missing_generalizations( - model: str, provider: str -) -> dict[str, object] | None: # mutable-ok: preserve the existing dict return contract +def match_fill_missing_generalizations(model: str, provider: str) -> Mapping[str, object] | None: """Return flagged capability rules matching ``model`` for ``provider``. Later rules override earlier ones on key conflicts. Only rules listing From 0f846cecb8566c2fea3e35bb948349fa3c4f7a07 Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 18:00:15 +0000 Subject: [PATCH 10/12] fix(model_info): single-assign fill_missing_for_providers in rule compiler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 25ea1bff613..5565e557392 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -112,6 +112,22 @@ class _CapabilityRule: _CompiledRule = _RoutingRule | _CapabilityRule +def _parse_fill_missing_for_providers(rule: Mapping[str, object], pattern_label: object) -> frozenset[str] | None: + if FILL_MISSING_FOR_PROVIDERS_FIELD not in rule: + return frozenset() + raw_fill_missing_for_providers: Final = rule.get(FILL_MISSING_FOR_PROVIDERS_FIELD) + if not isinstance(raw_fill_missing_for_providers, (list, tuple)) or not all( + isinstance(provider, str) for provider in raw_fill_missing_for_providers + ): + verbose_logger.warning( + "LiteLLM: skipping malformed fallback generalization rule %s ('%s' must be a list of provider strings).", + rule.get(NAME_FIELD, pattern_label), + FILL_MISSING_FOR_PROVIDERS_FIELD, + ) + return None + return frozenset(raw_fill_missing_for_providers) + + def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: if not isinstance(rule, dict): return () @@ -134,21 +150,9 @@ def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: e, ) return () - if FILL_MISSING_FOR_PROVIDERS_FIELD not in rule: - fill_missing_for_providers: Final[frozenset[str]] = frozenset() - else: - raw_fill_missing_for_providers: Final = rule.get(FILL_MISSING_FOR_PROVIDERS_FIELD) - if not isinstance(raw_fill_missing_for_providers, (list, tuple)) or not all( - isinstance(provider, str) for provider in raw_fill_missing_for_providers - ): - verbose_logger.warning( - "LiteLLM: skipping malformed fallback generalization rule %s " - "('%s' must be a list of provider strings).", - rule.get(NAME_FIELD, pattern), - FILL_MISSING_FOR_PROVIDERS_FIELD, - ) - return () - fill_missing_for_providers = frozenset(raw_fill_missing_for_providers) + fill_missing_for_providers: Final = _parse_fill_missing_for_providers(rule, pattern) + if fill_missing_for_providers is None: + return () if PROVIDER_KEY not in model_info: return ( _CapabilityRule( From 4cf20f31ed0067eb97b53b11bbc1df7815a01cd9 Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 18:39:27 +0000 Subject: [PATCH 11/12] refactor(model_info): drop types import from fallback_generalizations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/fallback_generalizations.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 5565e557392..be432d1fdd4 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -55,7 +55,6 @@ import logging import re from collections.abc import Mapping from dataclasses import dataclass -from types import MappingProxyType from typing import Final verbose_logger: Final = logging.getLogger("LiteLLM") @@ -225,9 +224,9 @@ class _FallbackGeneralizations: ) if not matched: return None - fill_missing: Final[Mapping[str, object]] = MappingProxyType( - {key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY} - ) + fill_missing: Final[Mapping[str, object]] = { + key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY + } return fill_missing or None From 250ff03a04ce260ed75d26e28bede2f64091c1f1 Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 14 Sep 2026 18:53:22 +0000 Subject: [PATCH 12/12] feat(model_info): scope fill_missing rules to azure, bedrock and vertex hosts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 8 ++++---- model_prices_and_context_window.json | 8 ++++---- .../test_fallback_generalizations.py | 16 +++++++++------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c9e85c28420..685e63dcc56 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic"], + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_for_providers": ["anthropic"], + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic"], + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_for_providers": ["openai"], + "fill_missing_for_providers": ["azure", "azure_ai", "openai"], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c9e85c28420..685e63dcc56 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57717,7 +57717,7 @@ { "name": "claude-adaptive-thinking", "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic"], + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true @@ -57726,7 +57726,7 @@ { "name": "claude-legacy-thinking", "pattern": "claude-[a-z]+-4[-._]6(?!\\d)", - "fill_missing_for_providers": ["anthropic"], + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], "description": "Claude at version 4.6 exactly, in any id shape that contains claude--4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.", "model_info": { "supports_legacy_thinking": true @@ -57743,7 +57743,7 @@ { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "fill_missing_for_providers": ["anthropic"], + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true @@ -57760,7 +57760,7 @@ { "name": "openai-reasoning-family-baseline", "pattern": "^(?!.*search-api)(?:[a-z0-9_.-]+/)*(?:ft:)?(?:o[1-9]\\d*(?![a-z0-9])|gpt-[5-9](?:\\.\\d+)?(?![0-9.])|(?:gpt-\\d+(?:\\.\\d+)?(?:-[a-z0-9]+)*-)?(?:codex|deep-research|chat-latest)(?![a-z0-9]))", - "fill_missing_for_providers": ["openai"], + "fill_missing_for_providers": ["azure", "azure_ai", "openai"], "description": "OpenAI reasoning families by id shape, under any provider namespace and with an optional ft: prefix: the o-series (o1, o3-pro, o4-mini), gpt-5 through gpt-9 majors including dotted minors and suffixed variants (gpt-5.5-cyber, gpt-6-astra), and the codex, deep-research and chat-latest lines when standalone or on a gpt base. gpt-5-search-api is excluded because it is a search-only surface. Every model here is a reasoning model, and the Responses API drops the caller's reasoning param for any mapped OpenAI model whose info lacks supports_reasoning, so an id the registry has not named yet keeps its reasoning settings instead of silently losing them. Rules lose to exact entries. Carries no mode and no pricing, so cost stays on the standard unpriced behavior.", "model_info": { "supports_reasoning": true diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 09870faaaf7..b2cc3ebe4c6 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -872,21 +872,23 @@ def test_shipped_openai_reasoning_rule_loses_to_mapped_entries(shipped_cost_map) @pytest.mark.parametrize( - "model,provider", + "model,provider,expected_supports_reasoning", [ - ("azure/us/o1-2024-12-17", "azure"), - ("github_copilot/gpt-5", "github_copilot"), - ("openrouter/openai/o1", "openrouter"), - ("perplexity/openai/gpt-5.4-mini", "perplexity"), + ("azure/us/o1-2024-12-17", "azure", True), + ("github_copilot/gpt-5", "github_copilot", None), + ("openrouter/openai/o1", "openrouter", None), + ("perplexity/openai/gpt-5.4-mini", "perplexity", None), ], ) -def test_shipped_openai_reasoning_rule_does_not_backfill_other_providers(shipped_cost_map, model, provider): +def test_shipped_openai_reasoning_rule_backfills_only_approved_providers( + shipped_cost_map, model, provider, expected_supports_reasoning +): assert model in litellm.model_cost raw_entry = litellm.model_cost[model] assert "supports_reasoning" not in raw_entry model_without_provider = model.removeprefix(f"{provider}/") info = litellm.get_model_info(model=model_without_provider, custom_llm_provider=provider) - assert info.get("supports_reasoning") is None + assert info.get("supports_reasoning") is expected_supports_reasoning assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0)