From 42c708670d9977219ecbcec92bf2fc56cf1b377c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 04:22:41 +0000 Subject: [PATCH] fix(model_info): guard backfill by mode, drop provider key, tighten claude major regex Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../fallback_generalizations.py | 5 ++- ...odel_prices_and_context_window_backup.json | 8 ++-- litellm/utils.py | 3 +- model_prices_and_context_window.json | 8 ++-- .../test_fallback_generalizations.py | 44 +++++++++++++++++++ 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index a5360309a70..8a029f9a642 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -191,7 +191,10 @@ class _FallbackGeneralizations: matched = tuple(rule.model_info for rule in self.backfill_rules if rule.pattern.search(model) is not None) if not matched: return None - return {key: value for model_info in matched for key, value in model_info.items()} + backfill: Final = { + key: value for model_info in matched for key, value in model_info.items() if key != PROVIDER_KEY + } + return backfill or None _registry: Final = _FallbackGeneralizations() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ba3c240a94..a06688faa5b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57716,9 +57716,9 @@ }, { "name": "claude-adaptive-thinking", - "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true } @@ -57742,9 +57742,9 @@ }, { "name": "claude-mid-conversation-system", - "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true } diff --git a/litellm/utils.py b/litellm/utils.py index 87f270fa757..74e7001effe 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -255,6 +255,7 @@ from litellm.types.utils import ( ) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} +_BACKFILL_MODES: Final = frozenset({"chat", "responses"}) # +-----------------------------------------------+ # | | @@ -5820,7 +5821,7 @@ def _get_model_info_helper( ): _model_info = None - if _model_info is not None and key is not None: + if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: backfill: Final = match_backfill_generalizations(key) if backfill is not None: _model_info = {**{k: v for k, v in backfill.items() if k not in _model_info}, **_model_info} diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6ba3c240a94..a06688faa5b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57716,9 +57716,9 @@ }, { "name": "claude-adaptive-thinking", - "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Turns on adaptive thinking for new versions and new families with no code change.", "model_info": { "supports_adaptive_thinking": true } @@ -57742,9 +57742,9 @@ }, { "name": "claude-mid-conversation-system", - "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", "backfill_exact_entries": true, - "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", "model_info": { "supports_mid_conversation_system": true } diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 0221b9bbffc..036abedd90c 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -139,6 +139,18 @@ def test_backfill_requires_per_rule_opt_in(restore_generalizations): ) assert match_backfill_generalizations("acme-1") is None + restore_generalizations( + [ + { + "name": "mixed", + "pattern": r"^acme-", + "backfill_exact_entries": True, + "model_info": {"litellm_provider": "openai", "supports_vision": True}, + } + ] + ) + assert match_backfill_generalizations("acme-1") == {"supports_vision": True} + restore_generalizations( [ { @@ -355,6 +367,12 @@ def test_exact_entries_backfill_only_missing_fields(restore_generalizations, mon "litellm_provider": "openai", "mode": "chat", }, + "acme-image": { + "input_cost_per_token": 5e-6, + "output_cost_per_token": 6e-6, + "litellm_provider": "openai", + "mode": "image_generation", + }, }, ) restore_generalizations( @@ -379,6 +397,9 @@ def test_exact_entries_backfill_only_missing_fields(restore_generalizations, mon assert bare["input_cost_per_token"] == 3e-6 assert bare["key"] == "acme-bare" + image = litellm.get_model_info("acme-image", custom_llm_provider="openai") + assert image.get("supports_reasoning") is None + restore_generalizations( [{"name": "acme-backfill", "pattern": r"^acme-", "model_info": {"supports_reasoning": True, "max_tokens": 5}}] ) @@ -503,6 +524,18 @@ def test_shipped_version_boundaries(shipped_cost_map, model, provider, adaptive, assert info.get("supports_mid_conversation_system") is mid_conversation, model +def test_shipped_claude_version_regex_excludes_undelimited_41(shipped_cost_map): + unmatched = match_capability_generalizations("github_copilot/claude-opus-41") + assert unmatched is None or "supports_adaptive_thinking" not in unmatched + assert unmatched is None or "supports_mid_conversation_system" not in unmatched + + for model in ("claude-opus-5", "claude-sonnet-4-8"): + matched = match_capability_generalizations(model) + assert matched is not None + assert matched["supports_adaptive_thinking"] is True + assert matched["supports_mid_conversation_system"] is True + + def test_shipped_rules_cover_new_families_like_fable_at_5_plus(shipped_cost_map): """Both version gates accept any claude-- id at major 5 or higher, bare major or major-minor, so a new family shaped like claude-fable-5 gets adaptive @@ -833,6 +866,17 @@ def test_shipped_openai_reasoning_rule_backfills_mapped_entries(shipped_cost_map assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0) +def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map): + model = "gemini/deep-research-pro-preview-12-2025" + assert model in litellm.model_cost + raw_entry = litellm.model_cost[model] + assert "supports_reasoning" not in raw_entry + assert raw_entry["mode"] == "image_generation" + + info = litellm.get_model_info("deep-research-pro-preview-12-2025", custom_llm_provider="gemini") + assert info.get("supports_reasoning") is None + + def test_shipped_claude_thinking_rules_backfill_without_family_limits(shipped_cost_map): model = "perplexity/anthropic/claude-sonnet-4-6" assert model in litellm.model_cost