scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 35

refactor(fallback-generalizations): split rules into routing and provider-neutral capability kinds

Browse source
This commit is contained in:
mateo-berri 2026-07-10 20:41:35 -07:00
parent 0c23c40627
commit 77885779ca
10 changed files with 474 additions and 411 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
basedpyright-code-budget.json
View file

@ -57,7 +57,7 @@
"limit": 5900
},
"reportMissingTypeArgument": {
"limit": 15918
"limit": 15913
},
"reportMissingTypeStubs": {
"limit": 41
@ -105,13 +105,13 @@
"limit": 113
},
"reportUnknownMemberType": {
"limit": 40541
"limit": 40539
},
"reportUnknownParameterType": {
"limit": 20418
"limit": 20413
},
"reportUnknownVariableType": {
"limit": 32151
"limit": 32144
},
"reportUnnecessaryCast": {
"limit": 177
@ -123,7 +123,7 @@
"limit": 7
},
"reportUnnecessaryIsInstance": {
"limit": 1212
"limit": 1211
},
"reportUntypedBaseClass": {
"limit": 165

209
litellm/litellm_core_utils/fallback_generalizations.py
View file

@ -3,151 +3,166 @@ Declarative fallback generalizations for unknown / newly-released models.
The ``fallback_generalizations`` block in ``model_prices_and_context_window.json``
holds an ordered list of rules. Each rule pairs a single case-insensitive regex
with the metadata to apply when a model name has no exact entry in the cost map.
The metadata is a partial cost-map entry: ``litellm_provider`` drives provider
routing, and the remaining fields (``mode``, ``supports_*``, context window,
pricing, ...) drive ``get_model_info`` / ``supports_*``.
with a ``model_info`` dict, and the structure of ``model_info`` decides which of
two kinds the rule is.
Precedence: rules are evaluated in file order and the first match wins. Callers
with extra constraints (model-info resolution checks the provider) use
``match_all_fallback_generalizations`` to skip inapplicable earlier rules instead
of discarding the model name. Rules are consulted only after exact and
case-insensitive lookups miss, so an exact entry always takes precedence over a
rule.
A ROUTING rule carries exactly one ``model_info`` key, ``litellm_provider``. It is
consumed only by ``get_llm_provider`` bare-id inference: the first routing rule
whose regex matches decides the provider. Routing rules never contribute to model
info.
A CAPABILITY rule carries any ``model_info`` keys except ``litellm_provider``
(``mode``, ``supports_*``, context window, pricing, ...). It is consumed by
``get_model_info`` fallback resolution: the ``model_info`` of ALL capability rules
whose regex matches is unioned in file order, with later rules overriding earlier
ones on key conflicts, and the caller backfills ``litellm_provider`` with the
provider it requested. If no capability rule matches, model-info resolution misses
as if no rules existed.
A rule that mixes ``litellm_provider`` with other ``model_info`` keys is invalid:
``set_fallback_generalizations`` logs a warning and skips it at install time (a
warning rather than a crash, because released proxies fetch this JSON remotely).
Rules are only consulted after exact and case-insensitive lookups miss, so an
exact cost-map entry always takes precedence over any rule.
Patterns are matched case-insensitively with ``re.search`` and are not implicitly
anchored: a rule must include ``^`` and ``$`` (as the shipped rules do) to bind to
the whole model name, otherwise it matches as a substring. Keeping anchoring in the
regex makes the rule the single, self-contained source of truth for what it matches.
A rule may set ``extends`` to the ``name`` of another rule to inherit that rule's
``model_info``; the rule's own ``model_info`` overrides the inherited keys, so a
narrow rule (for example a version-gated capability flag) carries only its delta
instead of duplicating the parent's pricing block. Inheritance is resolved once,
at install time, against each rule's raw (unresolved) ``model_info``; it is a
single level (a parent that itself extends is not chained).
anchored: a rule must include ``^`` and ``$`` to bind to the whole model name,
otherwise it matches as a substring. Keeping anchoring in the regex makes the rule
the single, self-contained source of truth for what it matches.
Any other keys on a rule (for example a free-text ``description`` documenting what
the regex matches) are ignored by the engine and exist only for the reader.
The compiled-regex list is built once and cached. ``match_fallback_generalization``
is O(number of rules); callers must only invoke it on a cache miss.
Rules are compiled and classified once, at install time. The match functions are
O(number of rules); callers must only invoke them on a cache miss.
"""
import re
from typing import Optional
from dataclasses import dataclass
from typing import Optional, Union
from litellm._logging import verbose_logger
NAME_FIELD = "name"
PATTERN_FIELD = "pattern"
MODEL_INFO_FIELD = "model_info"
EXTENDS_FIELD = "extends"
PROVIDER_KEY = "litellm_provider"
def _resolve_extends(rules: list) -> list:
"""Expand ``extends`` inheritance so each rule's ``model_info`` is self-contained.
@dataclass(frozen=True, slots=True)
class _RoutingRule:
pattern: re.Pattern
provider: str
A rule with ``extends: <name>`` is rewritten with ``model_info`` set to the parent's
``model_info`` overlaid by its own. Resolution is single-level and uses each rule's
raw ``model_info`` as the parent source. Non-dict rules and dangling parents are
passed through unchanged.
"""
base_by_name = {
rule[NAME_FIELD]: rule[MODEL_INFO_FIELD]
for rule in rules
if isinstance(rule, dict)
and isinstance(rule.get(NAME_FIELD), str)
and isinstance(rule.get(MODEL_INFO_FIELD), dict)
}
def resolved(rule: dict) -> dict:
parent_name = rule.get(EXTENDS_FIELD)
own_info = rule.get(MODEL_INFO_FIELD)
parent_info = base_by_name.get(parent_name) if isinstance(parent_name, str) else None
if parent_info is None or not isinstance(own_info, dict):
return rule
return {**rule, MODEL_INFO_FIELD: {**parent_info, **own_info}}
@dataclass(frozen=True, slots=True)
class _CapabilityRule:
pattern: re.Pattern
model_info: dict
return [resolved(rule) if isinstance(rule, dict) else rule for rule in rules]
_CompiledRule = Union[_RoutingRule, _CapabilityRule]
def _compile_rule(rule: object) -> Optional[_CompiledRule]:
if not isinstance(rule, dict):
return None
pattern = rule.get(PATTERN_FIELD)
model_info = rule.get(MODEL_INFO_FIELD)
if not isinstance(pattern, str) or not isinstance(model_info, dict):
verbose_logger.warning(
"LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').",
rule.get(NAME_FIELD, pattern),
PATTERN_FIELD,
MODEL_INFO_FIELD,
)
return None
try:
compiled = re.compile(pattern, re.IGNORECASE)
except re.error as e:
verbose_logger.warning(
"LiteLLM: skipping fallback generalization rule with invalid regex %r: %s",
pattern,
e,
)
return None
if PROVIDER_KEY not in model_info:
return _CapabilityRule(pattern=compiled, model_info=model_info)
provider = model_info[PROVIDER_KEY]
if len(model_info) == 1 and isinstance(provider, str):
return _RoutingRule(pattern=compiled, provider=provider)
verbose_logger.warning(
"LiteLLM: skipping invalid fallback generalization rule %s: a routing rule's '%s' must contain "
"'%s' as its only key (a string), and a capability rule must not contain '%s' at all.",
rule.get(NAME_FIELD, pattern),
MODEL_INFO_FIELD,
PROVIDER_KEY,
PROVIDER_KEY,
)
return None
class _FallbackGeneralizations:
"""Holds the active rule list and its lazily-compiled regex cache."""
"""Holds the raw rule list and its install-time-compiled routing and capability rules."""
def __init__(self) -> None:
self.rules: list[dict] = []
self._compiled: Optional[list[tuple[re.Pattern, dict]]] = None
self.rules: list = []
self.routing_rules: tuple = ()
self.capability_rules: tuple = ()
def set_rules(self, rules: Optional[list[dict]]) -> None:
self.rules = rules if isinstance(rules, list) else []
self._compiled = None
def set_rules(self, rules: Optional[list]) -> None:
installed = rules if isinstance(rules, list) else []
compiled = tuple(c for c in (_compile_rule(rule) for rule in installed) if c is not None)
self.rules = installed
self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule))
self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule))
def _compile(self) -> list[tuple[re.Pattern, dict]]:
compiled: list[tuple[re.Pattern, dict]] = []
for rule in self.rules:
if not isinstance(rule, dict):
continue
pattern = rule.get(PATTERN_FIELD)
model_info = rule.get(MODEL_INFO_FIELD)
if not isinstance(pattern, str) or not isinstance(model_info, dict):
verbose_logger.warning(
"LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').",
rule.get("name", pattern),
PATTERN_FIELD,
MODEL_INFO_FIELD,
)
continue
try:
compiled.append((re.compile(pattern, re.IGNORECASE), model_info))
except re.error as e:
verbose_logger.warning(
"LiteLLM: skipping fallback generalization rule with invalid regex %r: %s",
pattern,
e,
)
return compiled
def matches(self, model: str) -> list[dict]:
def match_routing(self, model: str) -> Optional[str]:
if not model:
return []
if self._compiled is None:
self._compiled = self._compile()
return [dict(model_info) for pattern, model_info in self._compiled if pattern.search(model) is not None]
return None
return next(
(rule.provider for rule in self.routing_rules if rule.pattern.search(model) is not None),
None,
)
def match(self, model: str) -> Optional[dict]:
return next(iter(self.matches(model)), None)
def match_capabilities(self, model: str) -> Optional[dict]:
if not model:
return None
matched = tuple(rule.model_info for rule in self.capability_rules if rule.pattern.search(model) is not None)
if not matched:
return None
return {key: value for model_info in matched for key, value in model_info.items()}
_registry = _FallbackGeneralizations()
def set_fallback_generalizations(rules: Optional[list[dict]]) -> None:
"""Install the active rule list and invalidate the compiled-regex cache.
def set_fallback_generalizations(rules: Optional[list]) -> None:
"""Install the active rule list, compiling and classifying each rule.
``extends`` inheritance is resolved here, once, before the rules are stored.
Malformed, invalid-regex, and mixed-kind rules are warned about and skipped here.
Called once when the model cost map is loaded (and again on any reload).
"""
_registry.set_rules(_resolve_extends(rules) if isinstance(rules, list) else rules)
_registry.set_rules(rules)
def get_fallback_generalization_rules() -> list[dict]:
def get_fallback_generalization_rules() -> list:
"""Return the raw rule list (read-only view for callers/tests)."""
return _registry.rules
def match_fallback_generalization(model: str) -> Optional[dict]:
"""Return the ``model_info`` of the first rule whose regex matches ``model``.
def match_routing_generalization(model: str) -> Optional[str]:
"""Return the provider of the first routing rule whose regex matches ``model``.
O(number of rules). Only call this once exact lookups have missed.
"""
return _registry.match(model)
return _registry.match_routing(model)
def match_all_fallback_generalizations(model: str) -> list[dict]:
"""Return the ``model_info`` of every rule whose regex matches ``model``, in rule order.
def match_capability_generalizations(model: str) -> Optional[dict]:
"""Return the union of the ``model_info`` of every capability rule matching ``model``.
Lets a caller with extra constraints (e.g. a provider match) skip an
inapplicable earlier rule instead of discarding the whole candidate.
Later rules override earlier ones on key conflicts. Returns ``None`` when no
capability rule matches. O(number of rules); only call once exact lookups have missed.
"""
return _registry.matches(model)
return _registry.match_capabilities(model)

8
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -4,7 +4,7 @@ from urllib.parse import urlparse
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.litellm_core_utils.fallback_generalizations import (
match_fallback_generalization,
match_routing_generalization,
)
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.secret_managers.main import get_secret, get_secret_str
@ -474,12 +474,10 @@ def get_llm_provider(
custom_llm_provider = "sap"
# Last resort for an otherwise-unknown model: a declarative
# fallback-generalization rule (e.g. routes future claude-* to anthropic).
# fallback-generalization routing rule (e.g. routes future claude-* to anthropic).
# Exact provider matches above always win; this only runs on a miss.
if not custom_llm_provider:
generalization = match_fallback_generalization(model)
if generalization is not None:
custom_llm_provider = generalization.get("litellm_provider") or None
custom_llm_provider = match_routing_generalization(model)
if not custom_llm_provider:
if litellm.suppress_debug_info is False:

49
litellm/model_prices_and_context_window_backup.json
View file

@ -45050,31 +45050,26 @@
"fallback_generalizations": {
"rules": [
{
"name": "bedrock-anthropic-claude-mid-conversation-system",
"pattern": "anthropic\\.claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Bedrock Invoke ids for Claude 4.8 or higher: anthropic.claude-<family> with minor 4.8 through 4.99, any 5.x or later major-minor, or a bare 5+ major, which also admits new families such as fable. These models accept mid-conversation role system messages in place (verified live on Opus 4.8, Sonnet 5 and Fable 5), so unmapped future Bedrock Claudes keep the cache-preserving in-place handling instead of the hoist-all default. Listed first so bare-id provider inference, which takes the first pattern hit, resolves these Bedrock ids to bedrock; model-info resolution skips provider-mismatched rules either way.",
"extends": "anthropic-claude",
"name": "bedrock-claude-ids",
"pattern": "anthropic\\.claude-",
"description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.",
"model_info": {
"litellm_provider": "bedrock",
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true
"litellm_provider": "bedrock"
}
},
{
"name": "anthropic-claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.",
"extends": "anthropic-claude",
"model_info": {
"supports_adaptive_thinking": true
}
},
{
"name": "anthropic-claude",
"name": "anthropic-claude-ids",
"pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"description": "A bare Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.",
"model_info": {
"litellm_provider": "anthropic"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...). Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"model_info": {
"litellm_provider": "anthropic",
"mode": "chat",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
@ -45090,6 +45085,22 @@
"supports_pdf_input": true,
"supports_system_messages": true
}
},
{
"name": "claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions with no code change.",
"model_info": {
"supports_adaptive_thinking": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.8 or higher, in any id shape that contains claude-<family>-: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.",
"model_info": {
"supports_mid_conversation_system": true
}
}
]
}

25
litellm/utils.py
View file

@ -61,7 +61,7 @@ from litellm._lazy_imports import (
)
from litellm._uuid import uuid
from litellm.litellm_core_utils.fallback_generalizations import (
match_all_fallback_generalizations,
match_capability_generalizations,
)
from litellm.constants import (
DEFAULT_CHAT_COMPLETION_PARAM_VALUES,
@ -5043,25 +5043,28 @@ def _get_model_info_from_generalization(
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
custom_llm_provider: Optional[str],
) -> Optional[tuple[str, dict]]:
"""Resolve an unmapped model via a declarative fallback-generalization rule.
"""Resolve an unmapped model via the declarative capability generalization rules.
Tries the same name candidates as the exact lookups, in the same order, and
returns ``(matched_name, model_info)`` for the first matching rule that also
satisfies the provider constraint; a rule scoped to another provider is
skipped in favor of later rules rather than discarding the candidate.
O(number of rules); only call after the exact lookups have missed.
returns ``(matched_name, model_info)`` for the first candidate matched by at
least one capability rule, with ``litellm_provider`` backfilled from the
provider the caller requested. O(number of rules); only call after the exact
lookups have missed.
"""
candidates = [
candidates = (
potential_model_names["combined_model_name"],
model,
potential_model_names["split_model"],
potential_model_names["combined_stripped_model_name"],
potential_model_names["stripped_model_name"],
]
)
for candidate in candidates:
for generalized_info in match_all_fallback_generalizations(candidate):
if _check_provider_match(model_info=generalized_info, custom_llm_provider=custom_llm_provider):
return candidate, generalized_info
generalized_info = match_capability_generalizations(candidate)
if generalized_info is None:
continue
if custom_llm_provider is None:
return candidate, generalized_info
return candidate, {**generalized_info, "litellm_provider": custom_llm_provider}
return None

49
model_prices_and_context_window.json
View file

@ -45283,31 +45283,26 @@
"fallback_generalizations": {
"rules": [
{
"name": "bedrock-anthropic-claude-mid-conversation-system",
"pattern": "anthropic\\.claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Bedrock Invoke ids for Claude 4.8 or higher: anthropic.claude-<family> with minor 4.8 through 4.99, any 5.x or later major-minor, or a bare 5+ major, which also admits new families such as fable. These models accept mid-conversation role system messages in place (verified live on Opus 4.8, Sonnet 5 and Fable 5), so unmapped future Bedrock Claudes keep the cache-preserving in-place handling instead of the hoist-all default. Listed first so bare-id provider inference, which takes the first pattern hit, resolves these Bedrock ids to bedrock; model-info resolution skips provider-mismatched rules either way.",
"extends": "anthropic-claude",
"name": "bedrock-claude-ids",
"pattern": "anthropic\\.claude-",
"description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.",
"model_info": {
"litellm_provider": "bedrock",
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true
"litellm_provider": "bedrock"
}
},
{
"name": "anthropic-claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.",
"extends": "anthropic-claude",
"model_info": {
"supports_adaptive_thinking": true
}
},
{
"name": "anthropic-claude",
"name": "anthropic-claude-ids",
"pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"description": "A bare Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.",
"model_info": {
"litellm_provider": "anthropic"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...). Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"model_info": {
"litellm_provider": "anthropic",
"mode": "chat",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
@ -45323,6 +45318,22 @@
"supports_pdf_input": true,
"supports_system_messages": true
}
},
{
"name": "claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions with no code change.",
"model_info": {
"supports_adaptive_thinking": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.8 or higher, in any id shape that contains claude-<family>-: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.",
"model_info": {
"supports_mid_conversation_system": true
}
}
]
}

10
tests/local_testing/test_get_llm_provider.py
View file

@ -509,10 +509,10 @@ def shipped_generalizations():
class TestClaudeModelPatternMatching:
"""
The ``anthropic-claude`` fallback generalization rule routes future Claude
models to the Anthropic provider without requiring a
The ``anthropic-claude-ids`` fallback generalization routing rule routes future
Claude models to the Anthropic provider without requiring a
model_prices_and_context_window.json entry. These tests exercise the rule
end-to-end through ``get_llm_provider`` and ``match_fallback_generalization``.
end-to-end through ``get_llm_provider`` and ``match_routing_generalization``.
"""
@pytest.mark.parametrize(
@ -556,10 +556,10 @@ class TestClaudeModelPatternMatching:
self, model, shipped_generalizations
):
from litellm.litellm_core_utils.fallback_generalizations import (
match_fallback_generalization,
match_routing_generalization,
)
assert match_fallback_generalization(model) is None
assert match_routing_generalization(model) is None
def test_routing_comes_from_the_rule_not_python(self, shipped_generalizations):
"""With the rule cleared, an unknown claude must no longer route to

472
tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
View file

@ -1,11 +1,13 @@
"""
Tests for the declarative fallback-generalizations mechanism.
Covers both the pure module (litellm.litellm_core_utils.fallback_generalizations)
and its end-to-end wiring into provider routing (get_llm_provider) and model-info
resolution (get_model_info / supports_*).
Covers the pure module (litellm.litellm_core_utils.fallback_generalizations): the
routing/capability rule split, install-time validation, capability unioning; and
its end-to-end wiring into provider routing (get_llm_provider) and model-info
resolution (get_model_info) including the shipped rules in the bundled cost map.
"""
import logging
import os
import sys
@ -14,10 +16,11 @@ import pytest
sys.path.insert(0, os.path.abspath("../../.."))
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
match_all_fallback_generalizations,
match_fallback_generalization,
match_capability_generalizations,
match_routing_generalization,
set_fallback_generalizations,
)
@ -32,92 +35,143 @@ def restore_generalizations():
set_fallback_generalizations(previous)
class _RecordingHandler(logging.Handler):
def __init__(self):
super().__init__(level=logging.WARNING)
self.messages = []
def emit(self, record):
self.messages.append(record.getMessage())
@pytest.fixture
def warning_messages():
handler = _RecordingHandler()
previous_level = verbose_logger.level
verbose_logger.setLevel(logging.WARNING)
verbose_logger.addHandler(handler)
try:
yield handler.messages
finally:
verbose_logger.removeHandler(handler)
verbose_logger.setLevel(previous_level)
# --------------------------------------------------------------------------- #
# Pure module behaviour
# Engine: routing rules
# --------------------------------------------------------------------------- #
def test_match_returns_model_info_of_first_matching_rule(restore_generalizations):
def test_routing_inference_first_match_wins(restore_generalizations):
restore_generalizations(
[
{"name": "first", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}},
{"name": "second", "pattern": r"^acme-pro-", "model_info": {"litellm_provider": "anthropic"}},
]
)
assert match_routing_generalization("acme-pro-1") == "openai"
assert match_routing_generalization("gpt-4o") is None
assert match_routing_generalization("") is None
def test_capability_rules_do_not_route(restore_generalizations):
restore_generalizations([{"name": "caps", "pattern": r"^acme-", "model_info": {"supports_vision": True}}])
assert match_routing_generalization("acme-pro-1") is None
def test_routing_match_is_case_insensitive(restore_generalizations):
restore_generalizations(
[{"name": "r", "pattern": r"^claude-opus", "model_info": {"litellm_provider": "anthropic"}}]
)
assert match_routing_generalization("CLAUDE-OPUS-9-9") == "anthropic"
# --------------------------------------------------------------------------- #
# Engine: capability rules
# --------------------------------------------------------------------------- #
def test_capability_union_is_last_wins_in_file_order(restore_generalizations):
restore_generalizations(
[
{
"name": "first",
"name": "broad",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "openai", "tag": "first"},
"model_info": {"mode": "chat", "supports_vision": True, "max_input_tokens": 1000},
},
{
"name": "second",
"name": "narrow",
"pattern": r"^acme-pro-",
"model_info": {"litellm_provider": "anthropic", "tag": "second"},
"model_info": {"supports_vision": False, "supports_reasoning": True},
},
]
)
# Both rules match "acme-pro-1"; first-in-list wins (documented precedence).
matched = match_fallback_generalization("acme-pro-1")
assert matched is not None
assert matched["tag"] == "first"
assert match_capability_generalizations("acme-pro-1") == {
"mode": "chat",
"supports_vision": False,
"max_input_tokens": 1000,
"supports_reasoning": True,
}
assert match_capability_generalizations("acme-basic-1") == {
"mode": "chat",
"supports_vision": True,
"max_input_tokens": 1000,
}
def test_match_all_returns_every_matching_rule_in_order(restore_generalizations):
def test_routing_rules_are_excluded_from_capability_results(restore_generalizations):
restore_generalizations(
[
{
"name": "first",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "openai", "tag": "first"},
},
{
"name": "second",
"pattern": r"^acme-pro-",
"model_info": {"litellm_provider": "anthropic", "tag": "second"},
},
{"name": "route", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}},
{"name": "caps", "pattern": r"^acme-pro-", "model_info": {"supports_vision": True}},
]
)
assert [m["tag"] for m in match_all_fallback_generalizations("acme-pro-1")] == ["first", "second"]
assert match_all_fallback_generalizations("gpt-4o") == []
assert match_capability_generalizations("acme-pro-1") == {"supports_vision": True}
assert match_capability_generalizations("acme-basic-1") is None
def test_provider_scoped_rule_is_skipped_for_other_providers(restore_generalizations):
"""Model-info resolution must fall through a provider-mismatched earlier rule to a
later applicable one, instead of discarding the model name at the first pattern hit."""
restore_generalizations(
[
{
"name": "bedrock-scoped",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "bedrock", "supports_vision": False},
},
{
"name": "openai-scoped",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "openai", "mode": "chat", "supports_vision": True},
},
]
)
litellm.get_model_info.cache_clear()
info = litellm.get_model_info("acme-fast-1", custom_llm_provider="openai")
assert info["litellm_provider"] == "openai"
assert info["supports_vision"] is True
def test_match_is_case_insensitive(restore_generalizations):
restore_generalizations(
[{"name": "r", "pattern": r"^claude-opus", "model_info": {"ok": True}}]
)
assert match_fallback_generalization("CLAUDE-OPUS-9-9") == {"ok": True}
def test_no_match_returns_none(restore_generalizations):
restore_generalizations(
[{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}]
)
assert match_fallback_generalization("gpt-4o") is None
assert match_fallback_generalization("") is None
def test_empty_rules_match_nothing(restore_generalizations):
def test_no_capability_match_returns_none(restore_generalizations):
restore_generalizations([{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}])
assert match_capability_generalizations("gpt-4o") is None
assert match_capability_generalizations("") is None
restore_generalizations([])
assert match_fallback_generalization("claude-opus-9-9") is None
assert match_capability_generalizations("claude-opus-9-9") is None
def test_reinstalling_rules_replaces_compiled_rules(restore_generalizations):
restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}])
assert match_capability_generalizations("aaa-1") == {"v": 1}
set_fallback_generalizations([{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}])
assert match_capability_generalizations("aaa-1") is None
assert match_capability_generalizations("bbb-1") == {"v": 2}
# --------------------------------------------------------------------------- #
# Engine: install-time validation
# --------------------------------------------------------------------------- #
def test_mixed_kind_rule_warns_and_is_skipped(restore_generalizations, warning_messages):
restore_generalizations(
[
{
"name": "mixed-kind",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "anthropic", "supports_vision": True},
},
{"name": "good-caps", "pattern": r"^acme-pro-", "model_info": {"supports_reasoning": True}},
]
)
assert any("mixed-kind" in message and "litellm_provider" in message for message in warning_messages)
assert match_routing_generalization("acme-pro-1") is None
assert match_capability_generalizations("acme-pro-1") == {"supports_reasoning": True}
def test_non_string_provider_rule_warns_and_is_skipped(restore_generalizations, warning_messages):
restore_generalizations([{"name": "bad-provider", "pattern": r"^acme-", "model_info": {"litellm_provider": 42}}])
assert any("bad-provider" in message for message in warning_messages)
assert match_routing_generalization("acme-1") is None
assert match_capability_generalizations("acme-1") is None
def test_malformed_rules_are_skipped_not_fatal(restore_generalizations):
@ -131,69 +185,7 @@ def test_malformed_rules_are_skipped_not_fatal(restore_generalizations):
{"name": "good", "pattern": r"^claude-", "model_info": {"good": True}},
]
)
# Non-dict entries and dicts with bad fields are all skipped; the one
# valid rule still matches.
assert match_fallback_generalization("claude-opus-9-9") == {"good": True}
def test_setting_rules_invalidates_compiled_cache(restore_generalizations):
restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}])
assert match_fallback_generalization("aaa-1") == {"v": 1}
# Re-install different rules; the compiled cache must be rebuilt.
set_fallback_generalizations(
[{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}]
)
assert match_fallback_generalization("aaa-1") is None
assert match_fallback_generalization("bbb-1") == {"v": 2}
def test_extends_inherits_parent_and_own_overrides(restore_generalizations):
"""A rule's ``extends`` pulls in the parent's model_info; its own keys win on conflict,
so a narrow rule carries only its delta instead of duplicating the parent."""
restore_generalizations(
[
{
"name": "base",
"pattern": r"^base-only$",
"model_info": {
"litellm_provider": "anthropic",
"input_cost_per_token": 5e-06,
"supports_vision": True,
},
},
{
"name": "child",
"pattern": r"^kid-",
"extends": "base",
"model_info": {
"supports_adaptive_thinking": True,
"supports_vision": False,
},
},
]
)
matched = match_fallback_generalization("kid-1")
assert matched == {
"litellm_provider": "anthropic",
"input_cost_per_token": 5e-06,
"supports_vision": False,
"supports_adaptive_thinking": True,
}
def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalizations):
"""A dangling ``extends`` is non-fatal: the rule resolves to its own model_info."""
restore_generalizations(
[
{
"name": "orphan",
"pattern": r"^orphan-",
"extends": "does-not-exist",
"model_info": {"litellm_provider": "openai"},
}
]
)
assert match_fallback_generalization("orphan-1") == {"litellm_provider": "openai"}
assert match_capability_generalizations("claude-opus-9-9") == {"good": True}
# --------------------------------------------------------------------------- #
@ -201,91 +193,61 @@ def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalization
# --------------------------------------------------------------------------- #
@pytest.fixture
def myco_rule(restore_generalizations):
"""A self-contained rule carrying provider, pricing, context and capabilities."""
def test_unknown_model_routes_via_routing_rule(restore_generalizations):
restore_generalizations([{"name": "myco", "pattern": r"^myco-", "model_info": {"litellm_provider": "openai"}}])
_, provider, _, _ = litellm.get_llm_provider(model="myco-fast-1")
assert provider == "openai"
def test_capability_info_backfills_requested_provider(restore_generalizations):
restore_generalizations(
[
{
"name": "myco",
"pattern": r"^myco-[a-z]+-\d+$",
"name": "beeco-caps",
"pattern": r"^beeco-[a-z]+-\d+$",
"model_info": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"max_input_tokens": 12345,
"max_output_tokens": 678,
"supports_vision": True,
"supports_function_calling": True,
},
}
]
)
return "myco-fast-1"
def test_unknown_model_routes_via_rule(myco_rule):
_, provider, _, _ = litellm.get_llm_provider(model=myco_rule)
assert provider == "openai"
def test_unknown_model_gets_pricing_context_and_capabilities(myco_rule):
info = litellm.get_model_info(myco_rule)
assert info["litellm_provider"] == "openai"
assert info["input_cost_per_token"] == 1e-06
assert info["output_cost_per_token"] == 2e-06
litellm.get_model_info.cache_clear()
info = litellm.get_model_info("beeco-fast-1", custom_llm_provider="groq")
assert info["litellm_provider"] == "groq"
assert info["max_input_tokens"] == 12345
assert info["supports_vision"] is True
other = litellm.get_model_info("beeco-fast-1", custom_llm_provider="openai")
assert other["litellm_provider"] == "openai"
def test_supports_helper_reads_through_generalization(myco_rule):
assert litellm.supports_vision(myco_rule) is True
assert litellm.supports_function_calling(myco_rule) is True
def test_routing_only_match_does_not_resolve_model_info(restore_generalizations):
restore_generalizations([{"name": "route", "pattern": r"^ceeco-", "model_info": {"litellm_provider": "openai"}}])
litellm.get_model_info.cache_clear()
with pytest.raises(Exception):
litellm.get_model_info("ceeco-fast-1", custom_llm_provider="openai")
def test_exact_entry_takes_precedence_over_rule(restore_generalizations):
"""An exact cost-map entry must win over a rule that also matches it."""
restore_generalizations(
[
{
"name": "shadow-gpt4o",
"pattern": r"^gpt-4o$",
"model_info": {
"litellm_provider": "anthropic",
"input_cost_per_token": 999.0,
},
}
]
[{"name": "shadow-gpt4o", "pattern": r"^gpt-4o$", "model_info": {"input_cost_per_token": 999.0}}]
)
litellm.get_model_info.cache_clear()
info = litellm.get_model_info("gpt-4o")
# Resolved from the real exact entry, not the shadowing rule.
assert info["litellm_provider"] == "openai"
assert info["input_cost_per_token"] != 999.0
def test_unknown_model_without_matching_rule_still_unmapped(restore_generalizations):
restore_generalizations(
[
{
"name": "claude",
"pattern": r"^claude-",
"model_info": {"litellm_provider": "anthropic"},
}
]
)
with pytest.raises(Exception):
litellm.get_model_info("totally-unknown-model-xyz")
# --------------------------------------------------------------------------- #
# Shipped anthropic-claude rule
# Shipped rules (bundled cost map)
# --------------------------------------------------------------------------- #
@pytest.fixture
def shipped_cost_map(monkeypatch):
"""Activate the bundled cost map so the shipped anthropic-claude rule is installed."""
"""Activate the bundled cost map so the shipped rules are installed."""
original_cost = litellm.model_cost
previous_rules = list(get_fallback_generalization_rules())
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
@ -299,40 +261,115 @@ def shipped_cost_map(monkeypatch):
set_fallback_generalizations(previous_rules)
def test_shipped_rule_marks_unmapped_high_version_claude_adaptive_without_pricing(
shipped_cost_map,
):
"""An unmapped Claude >= 4.6 resolves via the version-gated adaptive-thinking rule, which
inherits routing and capabilities from the base rule and adds ``supports_adaptive_thinking``.
The rule carries no pricing, so cost stays unpriced (zero, not a fabricated number) rather
than reporting a confidently-wrong price."""
model = "claude-opus-9-9"
def test_shipped_bare_claude_id_routes_to_anthropic(shipped_cost_map):
_, provider, _, _ = litellm.get_llm_provider(model="claude-haiku-4-6")
assert provider == "anthropic"
def test_shipped_bedrock_syntax_claude_id_routes_to_bedrock(shipped_cost_map):
"""Regression: a bedrock-syntax id must infer bedrock even when its version also
matches an unanchored Anthropic capability pattern. The old first-match-wins engine
routed global.anthropic.claude-haiku-4-6 to anthropic via the adaptive rule."""
for model in [
"global.anthropic.claude-haiku-4-6",
"us.anthropic.claude-haiku-4-6",
"anthropic.claude-haiku-4-6",
"eu.anthropic.claude-opus-5-0",
]:
assert model not in litellm.model_cost
_, provider, _, _ = litellm.get_llm_provider(model=model)
assert provider == "bedrock", model
def test_shipped_rules_resolve_unmapped_bedrock_claude_with_bedrock_provider(shipped_cost_map):
model = "us.anthropic.claude-haiku-4-6"
assert model not in litellm.model_cost
info = litellm.get_model_info(model)
assert info["litellm_provider"] == "anthropic"
info = litellm.get_model_info(model, custom_llm_provider="bedrock")
assert info["litellm_provider"] == "bedrock"
assert info["supports_adaptive_thinking"] is True
assert info["supports_function_calling"] is True
assert info["max_input_tokens"] == 200000
assert info.get("supports_mid_conversation_system") is None
assert not info.get("input_cost_per_token")
assert not info.get("output_cost_per_token")
def test_shipped_rule_resolves_unmapped_low_version_claude_without_adaptive(shipped_cost_map):
"""An unmapped Claude < 4.6 falls through to the version-neutral anthropic-claude rule: it
gets provider routing and baseline capabilities but no ``supports_adaptive_thinking`` flag,
so a sub-4.6 alias such as ``claude-opus-4-0`` resolves yet is never marked adaptive."""
model = "claude-opus-4-0"
def test_shipped_rules_stack_adaptive_and_mid_conversation_flags(shipped_cost_map):
model = "claude-opus-4-9"
assert model not in litellm.model_cost
info = litellm.get_model_info(model)
info = litellm.get_model_info(model, custom_llm_provider="anthropic")
assert info["litellm_provider"] == "anthropic"
assert info["supports_adaptive_thinking"] is True
assert info["supports_mid_conversation_system"] is True
assert info["supports_function_calling"] is True
@pytest.mark.parametrize(
"model,provider",
[
("claude-opus-4-9@20260101", "vertex_ai"),
("databricks-claude-opus-5-1", "databricks"),
],
)
def test_shipped_rules_are_provider_neutral_for_unmapped_ids(shipped_cost_map, model, provider):
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider=provider)
assert info["litellm_provider"] == provider
assert info["supports_adaptive_thinking"] is True
assert info["supports_mid_conversation_system"] is True
assert info["supports_function_calling"] is True
@pytest.mark.parametrize(
"model,provider,adaptive,mid_conversation",
[
("us.anthropic.claude-opus-4-5", "bedrock", None, None),
("claude-haiku-4-6", "anthropic", True, None),
("claude-haiku-4-7", "anthropic", True, None),
("claude-haiku-4-8", "anthropic", True, True),
("claude-haiku-4-9", "anthropic", True, True),
("claude-haiku-4-10", "anthropic", True, True),
("claude-haiku-5-0", "anthropic", True, True),
("claude-sonnet-5-1", "anthropic", True, True),
],
)
def test_shipped_version_boundaries(shipped_cost_map, model, provider, adaptive, mid_conversation):
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider=provider)
assert info["litellm_provider"] == provider
assert info["supports_function_calling"] is True
assert info.get("supports_adaptive_thinking") is None
assert not info.get("input_cost_per_token")
assert info.get("supports_adaptive_thinking") is adaptive, model
assert info.get("supports_mid_conversation_system") is mid_conversation, model
def test_shipped_mid_conversation_rule_covers_new_families_like_fable(shipped_cost_map):
"""The 4.8+ mid-conversation gate accepts bare 5+ majors, so a new family shaped
like claude-fable-5 is covered. The adaptive rule keeps its opus/sonnet/haiku
family list, so a fable-shaped id gets no adaptive flag from the rules."""
model = "claude-fable-5-1"
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider="anthropic")
assert info["supports_mid_conversation_system"] is True
assert info.get("supports_adaptive_thinking") is None
assert info["supports_function_calling"] is True
def test_shipped_exact_entry_beats_rules(shipped_cost_map):
model = "us.anthropic.claude-sonnet-4-6"
assert model in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider="bedrock")
assert info["litellm_provider"] == "bedrock_converse"
assert info["input_cost_per_token"] == 3.3e-06
assert info["max_input_tokens"] == 1000000
assert info["supports_adaptive_thinking"] is True
assert info.get("supports_mid_conversation_system") is None
def test_shipped_adaptive_rule_gates_on_version_not_pricing(shipped_cost_map):
"""The version-gated ``anthropic-claude-adaptive-thinking`` rule marks an unmapped
Claude adaptive only from >= 4.6, including provider-prefixed ids the anchored pricing
rule cannot match, while leaving < 4.6 (and the dated Opus 4.0 form) non-adaptive."""
"""The version-gated adaptive-thinking capability rule marks an unmapped Claude
adaptive only from >= 4.6, including provider-prefixed ids the anchored routing
rule cannot match, while leaving the dated Opus 4.0 form non-adaptive."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
adaptive = "us.anthropic.claude-opus-4-9"
@ -343,12 +380,10 @@ def test_shipped_adaptive_rule_gates_on_version_not_pricing(shipped_cost_map):
assert AnthropicModelInfo._is_adaptive_thinking_model(non_adaptive) is False
def test_shipped_bedrock_rule_resolves_unmapped_future_claude_for_bedrock(shipped_cost_map):
"""An unmapped Bedrock Claude >= 4.8 resolves via the bedrock-scoped
``bedrock-anthropic-claude-mid-conversation-system`` rule even when the lookup
carries ``custom_llm_provider="bedrock"``, which the provider check uses to drop
the anthropic-scoped rules. It inherits base capabilities, gains both
version-gated flags, and stays unpriced."""
def test_shipped_rules_resolve_unmapped_future_bedrock_claude_with_both_flags(shipped_cost_map):
"""An unmapped Bedrock Claude >= 4.8 resolves for custom_llm_provider="bedrock" with
baseline capabilities, both version-gated flags, the bedrock provider backfilled, and
no fabricated pricing."""
model = "us.anthropic.claude-opus-4-9"
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider="bedrock")
@ -359,11 +394,11 @@ def test_shipped_bedrock_rule_resolves_unmapped_future_claude_for_bedrock(shippe
assert not info.get("input_cost_per_token")
def test_shipped_bedrock_mid_conversation_rule_gates_on_version_and_naming(shipped_cost_map):
"""The bedrock rule only claims Bedrock-style ids at 4.8+, bare 5+ majors and
new families included; pre-4.8 Bedrock ids and native ids never gain the flag,
and the rule outranks the anthropic-scoped ones for Bedrock ids because it is
listed first."""
def test_shipped_mid_conversation_gate_on_bedrock_ids(shipped_cost_map):
"""Bedrock-syntax ids gain ``supports_mid_conversation_system`` only from 4.8 upward,
bare 5+ majors and new families included; 4.7-and-below Bedrock ids never gain it.
The flag comes from the provider-neutral capability rule rather than a bedrock-scoped
one, so the same gate covers native and vertex-shaped ids too."""
for flagged in (
"us.anthropic.claude-opus-4-8",
"jp.anthropic.claude-opus-4-8",
@ -371,17 +406,14 @@ def test_shipped_bedrock_mid_conversation_rule_gates_on_version_and_naming(shipp
"us.anthropic.claude-fable-5",
"anthropic.claude-sonnet-5-20260101-v1:0",
):
matched = match_fallback_generalization(flagged)
matched = match_capability_generalizations(flagged)
assert matched is not None, flagged
assert matched["litellm_provider"] == "bedrock", flagged
assert matched["supports_mid_conversation_system"] is True, flagged
for unflagged in (
"us.anthropic.claude-opus-4-7",
"us.anthropic.claude-sonnet-4-6",
"us.anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"claude-opus-4-9",
"claude-sonnet-5",
):
matched = match_fallback_generalization(unflagged)
matched = match_capability_generalizations(unflagged)
assert matched is None or not matched.get("supports_mid_conversation_system"), unflagged

49
tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py
View file

@ -13,7 +13,8 @@ sys.path.insert(0, os.path.abspath("../../.."))
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
match_fallback_generalization,
match_capability_generalizations,
match_routing_generalization,
set_fallback_generalizations,
)
from litellm.litellm_core_utils.get_model_cost_map import (
@ -102,9 +103,7 @@ def test_finalize_pops_key_and_installs_rules():
# The reserved key is removed from the returned model map ...
assert FALLBACK_GENERALIZATIONS_KEY not in finalized
# ... and its rules are installed into the generalizations module.
assert match_fallback_generalization("widget-9") == {
"litellm_provider": "openai"
}
assert match_routing_generalization("widget-9") == "openai"
finally:
set_fallback_generalizations(previous)
@ -116,27 +115,25 @@ def test_finalize_with_no_block_clears_rules():
[{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}]
)
_finalize_model_cost_map(_make_models(2))
assert match_fallback_generalization("x-1") is None
assert match_capability_generalizations("x-1") is None
finally:
set_fallback_generalizations(previous)
def test_shipped_backup_carries_the_anthropic_claude_rule():
"""The bundled backup must ship the anthropic-claude rule so a fresh install
(or an offline fallback) routes unknown Claude models without code changes."""
def test_shipped_backup_carries_the_claude_routing_rules():
"""The bundled backup must ship the Claude routing rules so a fresh install
(or an offline fallback) routes unknown Claude models without code changes.
Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", [])
names = {r.get("name") for r in rules}
assert "anthropic-claude" in names
rule = next(r for r in rules if r.get("name") == "anthropic-claude")
assert rule["model_info"]["litellm_provider"] == "anthropic"
names = [r.get("name") for r in rules]
assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids")
previous = list(get_fallback_generalization_rules())
try:
set_fallback_generalizations(rules)
matched = match_fallback_generalization("claude-opus-4-9")
assert matched is not None and matched["litellm_provider"] == "anthropic"
assert match_routing_generalization("claude-opus-4-9") == "anthropic"
assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock"
finally:
set_fallback_generalizations(previous)
@ -147,23 +144,19 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
route) and on the version-gated anthropic-claude-adaptive-thinking rule for
unmapped future Claudes, while leaving the dated Claude 4.0 names
("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor
version. The version-neutral anthropic-claude pricing rule must not flag it, so
an unmapped sub-4.6 name is priced but stays non-adaptive. The adaptive rule must
inherit pricing from the pricing rule via ``extends`` and carry only its delta, so
the Opus-tier price block is never duplicated across rules."""
version. The version-neutral claude-family-baseline capability rule must not flag
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
carries only its delta; capability unioning stacks it onto the baseline, so the
baseline block is never duplicated across rules and no rule needs ``extends``."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
pricing_rule = next(r for r in rules if r.get("name") == "anthropic-claude")
adaptive_rule = next(
r for r in rules if r.get("name") == "anthropic-claude-adaptive-thinking"
)
assert "supports_adaptive_thinking" not in pricing_rule["model_info"]
assert adaptive_rule["model_info"]["supports_adaptive_thinking"] is True
assert "extends" not in pricing_rule
assert adaptive_rule.get("extends") == "anthropic-claude"
baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline")
adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking")
assert "supports_adaptive_thinking" not in baseline_rule["model_info"]
assert "litellm_provider" not in baseline_rule["model_info"]
assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True}
assert all("extends" not in r for r in rules)
for adaptive in [
"anthropic.claude-opus-4-8",

4
type-discipline-budget.json
View file

@ -1,9 +1,9 @@
{
"LIT001": {
"limit": 23452
"limit": 23439
},
"LIT002": {
"limit": 27522
"limit": 27517
},
"LIT003": {
"limit": 292