From 1329036ca1cc8e3ec07274d02ff80af2cffa07b2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:00:41 -0700 Subject: [PATCH] fix(fallback-generalizations): tolerate legacy remote rule schema and keep register_model cache-pricing inheritance --- basedpyright-code-budget.json | 8 +- .../fallback_generalizations.py | 86 ++++++++++++++----- litellm/utils.py | 27 +++++- .../test_fallback_generalizations.py | 76 ++++++++++++++-- type-discipline-budget.json | 4 +- 5 files changed, 164 insertions(+), 37 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 8c5bf165c7e..c29adbd3044 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5900 }, "reportMissingTypeArgument": { - "limit": 15913 + "limit": 15908 }, "reportMissingTypeStubs": { "limit": 41 @@ -108,10 +108,10 @@ "limit": 40539 }, "reportUnknownParameterType": { - "limit": 20413 + "limit": 20408 }, "reportUnknownVariableType": { - "limit": 32144 + "limit": 32143 }, "reportUnnecessaryCast": { "limit": 177 @@ -123,7 +123,7 @@ "limit": 7 }, "reportUnnecessaryIsInstance": { - "limit": 1211 + "limit": 1210 }, "reportUntypedBaseClass": { "limit": 165 diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index c9b6d83e9f1..410bb9623fe 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -19,9 +19,18 @@ ones on key conflicts, and the caller backfills ``litellm_provider`` with the provider it requested. If no capability rule matches, model-info resolution misses as if no rules existed. -A rule that mixes ``litellm_provider`` with other ``model_info`` keys is invalid: -``set_fallback_generalizations`` logs a warning and skips it at install time (a -warning rather than a crash, because released proxies fetch this JSON remotely). +LEGACY-SCHEMA SHIM (temporary, until the new-schema JSON reaches main): released +proxies fetch this JSON remotely from main, whose block still ships the old schema +where a rule mixes ``litellm_provider`` with capability keys and may inherit a +parent's ``model_info`` via ``extends``. Such a legacy rule is tolerated rather +than skipped: ``extends`` is resolved once at install time (single level, against +raw parents), and the resolved rule acts as BOTH kinds, a routing rule (its +``litellm_provider`` participates in first-hit inference) and a capability rule +(its full ``model_info``, provider included, participates in the union). New-schema +rules never mix the two and never use ``extends``. A rule whose +``litellm_provider`` is not a string is invalid and is warned about and skipped +(a warning rather than a crash, for the same remote-fetch reason). + Rules are only consulted after exact and case-insensitive lookups miss, so an exact cost-map entry always takes precedence over any rule. @@ -47,6 +56,36 @@ NAME_FIELD = "name" PATTERN_FIELD = "pattern" MODEL_INFO_FIELD = "model_info" PROVIDER_KEY = "litellm_provider" +LEGACY_EXTENDS_FIELD = "extends" + + +def _resolve_legacy_extends(rules: list) -> list: + """Expand legacy ``extends`` inheritance so each rule's ``model_info`` is self-contained. + + Compatibility shim for the old remote schema: single level, resolved against each + parent's raw ``model_info``, with the child's own keys winning on conflict. Non-dict + rules and dangling parents pass through unchanged; new-schema rules carry no + ``extends`` and are untouched. + """ + base_by_name = { + rule[NAME_FIELD]: rule[MODEL_INFO_FIELD] + for rule in rules + if isinstance(rule, dict) + and isinstance(rule.get(NAME_FIELD), str) + and isinstance(rule.get(MODEL_INFO_FIELD), dict) + } + + def resolved(rule: object) -> object: + if not isinstance(rule, dict): + return rule + parent_name = rule.get(LEGACY_EXTENDS_FIELD) + own_info = rule.get(MODEL_INFO_FIELD) + parent_info = base_by_name.get(parent_name) if isinstance(parent_name, str) else None + if parent_info is None or not isinstance(own_info, dict): + return rule + return {**rule, MODEL_INFO_FIELD: {**parent_info, **own_info}} + + return [resolved(rule) for rule in rules] @dataclass(frozen=True, slots=True) @@ -64,9 +103,9 @@ class _CapabilityRule: _CompiledRule = Union[_RoutingRule, _CapabilityRule] -def _compile_rule(rule: object) -> Optional[_CompiledRule]: +def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: if not isinstance(rule, dict): - return None + return () pattern = rule.get(PATTERN_FIELD) model_info = rule.get(MODEL_INFO_FIELD) if not isinstance(pattern, str) or not isinstance(model_info, dict): @@ -76,7 +115,7 @@ def _compile_rule(rule: object) -> Optional[_CompiledRule]: PATTERN_FIELD, MODEL_INFO_FIELD, ) - return None + return () try: compiled = re.compile(pattern, re.IGNORECASE) except re.error as e: @@ -85,21 +124,24 @@ def _compile_rule(rule: object) -> Optional[_CompiledRule]: pattern, e, ) - return None + return () if PROVIDER_KEY not in model_info: - return _CapabilityRule(pattern=compiled, model_info=model_info) + return (_CapabilityRule(pattern=compiled, model_info=model_info),) provider = model_info[PROVIDER_KEY] - if len(model_info) == 1 and isinstance(provider, str): - return _RoutingRule(pattern=compiled, provider=provider) - verbose_logger.warning( - "LiteLLM: skipping invalid fallback generalization rule %s: a routing rule's '%s' must contain " - "'%s' as its only key (a string), and a capability rule must not contain '%s' at all.", - rule.get(NAME_FIELD, pattern), - MODEL_INFO_FIELD, - PROVIDER_KEY, - PROVIDER_KEY, + if not isinstance(provider, str): + verbose_logger.warning( + "LiteLLM: skipping invalid fallback generalization rule %s: '%s' in '%s' must be a string.", + rule.get(NAME_FIELD, pattern), + PROVIDER_KEY, + MODEL_INFO_FIELD, + ) + return () + if len(model_info) == 1: + return (_RoutingRule(pattern=compiled, provider=provider),) + return ( + _RoutingRule(pattern=compiled, provider=provider), + _CapabilityRule(pattern=compiled, model_info=model_info), ) - return None class _FallbackGeneralizations: @@ -112,7 +154,7 @@ class _FallbackGeneralizations: def set_rules(self, rules: Optional[list]) -> None: installed = rules if isinstance(rules, list) else [] - compiled = tuple(c for c in (_compile_rule(rule) for rule in installed) if c is not None) + compiled = tuple(kind for rule in _resolve_legacy_extends(installed) for kind in _compile_rule(rule)) self.rules = installed self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule)) self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule)) @@ -140,8 +182,10 @@ _registry = _FallbackGeneralizations() def set_fallback_generalizations(rules: Optional[list]) -> None: """Install the active rule list, compiling and classifying each rule. - Malformed, invalid-regex, and mixed-kind rules are warned about and skipped here. - Called once when the model cost map is loaded (and again on any reload). + Legacy ``extends`` inheritance is resolved here, once, before classification; + a legacy rule mixing ``litellm_provider`` with capability keys installs as both + kinds. Malformed and invalid-regex rules are warned about and skipped. Called + once when the model cost map is loaded (and again on any reload). """ _registry.set_rules(rules) diff --git a/litellm/utils.py b/litellm/utils.py index b25841e5389..907834dc5a4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2651,6 +2651,26 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> Optional[Dict[ return None +def _get_builtin_model_info_for_registration(model: str) -> Optional[ModelInfo]: + """Resolve ``model`` to its built-in cost-map entry for registration merging. + + Returns ``None`` when the lookup raises or when it resolved via a + fallback-generalization capability rule, detected as the resolved key missing + ``litellm.model_cost`` while matching a capability rule. A rule-derived entry + carries no pricing, so treating it as a hit would skip the built-in + cache-pricing inheritance for prefix-mangled keys. + """ + try: + info = get_model_info(model=model) + except Exception: + return None + if info["key"] in litellm.model_cost: + return info + if match_capability_generalizations(info["key"]) is None: + return info + return None + + def register_model(model_cost: Union[str, dict]): """ Register new / Override existing models (and their pricing) to specific providers. @@ -2691,10 +2711,11 @@ def register_model(model_cost: Union[str, dict]): existing_model = litellm.model_cost.get(key, {}) model_cost_key = key else: - try: - existing_model = cast(dict, get_model_info(model=key)) + builtin_model_info = _get_builtin_model_info_for_registration(model=_key_str) + if builtin_model_info is not None: + existing_model = cast(dict, builtin_model_info) model_cost_key = existing_model["key"] - except Exception: + else: existing_model = {} model_cost_key = key builtin_entry = _resolve_builtin_model_cost_entry(key=_key_str, provider=provider) diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index e4407b547cc..652f99ee402 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -147,24 +147,86 @@ def test_reinstalling_rules_replaces_compiled_rules(restore_generalizations): # --------------------------------------------------------------------------- # -# Engine: install-time validation +# Engine: install-time validation and legacy-schema shim # --------------------------------------------------------------------------- # -def test_mixed_kind_rule_warns_and_is_skipped(restore_generalizations, warning_messages): +def test_legacy_mixed_rule_acts_as_both_kinds(restore_generalizations): + """A legacy rule mixing ``litellm_provider`` with capability keys routes AND + contributes its full model_info (provider included) to the capability union.""" restore_generalizations( [ { - "name": "mixed-kind", + "name": "legacy-mixed", "pattern": r"^acme-", "model_info": {"litellm_provider": "anthropic", "supports_vision": True}, }, - {"name": "good-caps", "pattern": r"^acme-pro-", "model_info": {"supports_reasoning": True}}, + {"name": "new-caps", "pattern": r"^acme-pro-", "model_info": {"supports_reasoning": True}}, ] ) - assert any("mixed-kind" in message and "litellm_provider" in message for message in warning_messages) - assert match_routing_generalization("acme-pro-1") is None - assert match_capability_generalizations("acme-pro-1") == {"supports_reasoning": True} + assert match_routing_generalization("acme-pro-1") == "anthropic" + assert match_capability_generalizations("acme-pro-1") == { + "litellm_provider": "anthropic", + "supports_vision": True, + "supports_reasoning": True, + } + + +LEGACY_MAIN_RULES = [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": {"supports_adaptive_thinking": True}, + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": True, + "supports_parallel_function_calling": True, + "supports_vision": True, + "supports_tool_choice": True, + "supports_assistant_prefill": True, + "supports_prompt_caching": True, + "supports_response_schema": True, + "supports_reasoning": True, + "supports_pdf_input": True, + "supports_system_messages": True, + }, + }, +] + + +def test_legacy_main_schema_keeps_unmapped_claude_working(restore_generalizations): + """Pins the remote-map transition window: a released proxy running this engine + against main's old-schema block (mixed provider+capability rule plus ``extends``, + copied verbatim above) must keep unmapped-Claude inference and info resolution + working until the new-schema JSON reaches main.""" + restore_generalizations([dict(rule) for rule in LEGACY_MAIN_RULES]) + litellm.get_model_info.cache_clear() + + _, provider, _, _ = litellm.get_llm_provider(model="claude-opus-9-9") + assert provider == "anthropic" + + info = litellm.get_model_info("claude-opus-9-9") + assert info["litellm_provider"] == "anthropic" + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + assert info["max_input_tokens"] == 200000 + assert not info.get("input_cost_per_token") + + low = litellm.get_model_info("claude-opus-4-0") + assert low["litellm_provider"] == "anthropic" + assert low["supports_function_calling"] is True + assert low.get("supports_adaptive_thinking") is None def test_non_string_provider_rule_warns_and_is_skipped(restore_generalizations, warning_messages): diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 545f5ac0e94..6bc301baa0e 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23439 + "limit": 23424 }, "LIT002": { - "limit": 27517 + "limit": 27514 }, "LIT003": { "limit": 292