diff --git a/litellm/constants.py b/litellm/constants.py index 1300668cc70..8995b76c666 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -715,6 +715,7 @@ openai_compatible_endpoints: List = [ "https://api.clarifai.com/v2/ext/openai/v1", "https://api.libertai.io/v1", "https://pinstripes.io/v1", + "https://api.meta.ai/v1", ] @@ -781,6 +782,7 @@ openai_compatible_providers: List = [ "ragflow", "pinstripes", # Pinstripes - JSON-configured provider "darkbloom", + "meta", # Meta Model API (Muse Spark) - JSON-configured provider ] openai_text_completion_compatible_providers: List = [ # providers that support `/v1/completions` "together_ai", diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 17011bb8db7..3038bdb90b2 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -223,6 +223,15 @@ lives in [`plumbing/`](./plumbing): readers/exporters receive them alongside the server metrics, and one is built and registered as the global only when none is set (mirroring how V2 owns trace export). +- [`events.py`](./plumbing/events.py) — GenAI client events. Gated on + `enable_events` (`LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS`), a failed LLM call + records the semconv `gen_ai.client.operation.exception` log event at severity + WARN, carrying `exception.type` / `exception.message` / `exception.stacktrace` + and correlated to the failed span through the trace and span ids. The + `LoggerProvider` is resolved like the meter provider, except that an explicit + `NoOpLoggerProvider` global is an operator opt-out that builds no recorder at + all. The deprecated `error.*` span attributes and the `exception` span event + are still stamped by the emitter for backwards compatibility. ### Adapter diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 46aa166a8bb..f97f8b8394c 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -18,6 +18,7 @@ from litellm.integrations.otel.model.payloads import ( ServiceSpanData, SpanError, ) +from litellm.integrations.otel.plumbing.events import GenAIEventRecorder from litellm.integrations.otel.plumbing.providers import to_otel_span_kind from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError from litellm.integrations.otel.model.spans import ( @@ -77,9 +78,11 @@ class SpanEmitter: tracer: Tracer, config: OpenTelemetryV2Config, mappers: Sequence[AttributeMapper] | None = None, + event_recorder: GenAIEventRecorder | None = None, ) -> None: self._tracer = tracer self._config = config + self._event_recorder = event_recorder # The mapper chain is the sole source of span attributes. When not # passed in, resolve it from the config so there's one source of truth. self._mappers: list[AttributeMapper] = ( @@ -223,6 +226,14 @@ class SpanEmitter: ExceptionEvent.NAME, {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message}, ) + if self._event_recorder is not None and role is SpanRole.LLM_CALL: + self._event_recorder.record_operation_exception( + span_context=span.get_span_context(), + error_type=error_type, + message=message, + stack_trace=error.stack_trace, + timestamp_ns=end_time_ns, + ) # On success leave the status UNSET (the semconv default) rather than # forcing OK — that matches the FastAPI server span and avoids implying a # span-level health signal litellm doesn't actually evaluate. Only a diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index e258b239d93..be72fabd387 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, cast from opentelemetry.context import Context, attach, get_current +from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import Span, Tracer, get_current_span, use_span @@ -40,14 +41,17 @@ from litellm.integrations.otel.model.payloads import ( is_mcp_list_tools, is_mcp_tool_call, ) +from litellm.integrations.otel.plumbing.events import GenAIEventRecorder from litellm.integrations.otel.plumbing.metrics import ( GenAIMetricRecorder, create_genai_metrics, ) from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, + get_event_logger, get_meter, get_tracer, + resolve_logger_provider, resolve_meter_provider, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache @@ -104,7 +108,7 @@ class OpenTelemetryV2(CustomLogger): config: OpenTelemetryV2Config | None = None, callback_name: str | None = None, tracer_provider: TracerProvider | None = None, - logger_provider: Any | None = None, # reserved for OTel logs + logger_provider: LoggerProvider | None = None, meter_provider: Any | None = None, **kwargs: Any, ) -> None: @@ -117,7 +121,12 @@ class OpenTelemetryV2(CustomLogger): self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) self._metric_filter_error_logged = False - self._emitter = SpanEmitter(self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)) + self._emitter = SpanEmitter( + self.tracer, + self.config, + mappers=resolve_mappers(self.config.mapper_names), + event_recorder=self._init_events(logger_provider), + ) self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME) self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict() self._init_otel_logger_on_litellm_proxy() @@ -136,6 +145,22 @@ class OpenTelemetryV2(CustomLogger): meter = get_meter(provider, LITELLM_TRACER_NAME) return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name) + def _init_events(self, logger_provider: LoggerProvider | None) -> "GenAIEventRecorder | None": + """Create the GenAI event recorder when events are enabled, else ``None``. + + ``logger_provider`` is an explicit override (tests inject one); otherwise the + provider is resolved from the OTel global so an operator-configured logs + pipeline receives the events, building and registering one only when no + global provider is set. A ``None`` resolution means the operator opted out + of the logs signal, so no recorder is built. + """ + if not self.config.enable_events: + return None + provider = resolve_logger_provider(self.config, logger_provider) + if provider is None: + return None + return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME)) + # ====================================================================== # # Proxy global registration # ====================================================================== # diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 69d1e454655..c781ad34e87 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -180,6 +180,19 @@ class ExceptionEvent: NAME: Final = "exception" TYPE: Final = "exception.type" MESSAGE: Final = "exception.message" + STACKTRACE: Final = "exception.stacktrace" + + +class GenAIEvent: + """GenAI semconv event names, from the GenAI registry's *events* section. + + ``gen_ai.client.operation.exception`` is defined as a log-based event + (severity WARN) carrying the ``exception.*`` trio, correlated to the failed + span via the trace/span ids — the semconv-compliant home for GenAI failure + details, unlike the deprecated ``error.message`` span attribute. + """ + + OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception" class Server: diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py new file mode 100644 index 00000000000..f674526d04f --- /dev/null +++ b/litellm/integrations/otel/plumbing/events.py @@ -0,0 +1,52 @@ +"""GenAI client events: the ``gen_ai.client.operation.exception`` log event. + +The GenAI semantic conventions define exception recording for client +operations as a log-based event (severity WARN) carrying the ``exception.*`` +attribute trio, correlated to the failed span through the trace/span ids — +not as a span attribute or span event. This module owns building and +emitting that event; the exporter pipeline it rides is built in +:mod:`litellm.integrations.otel.plumbing.providers`. +""" + +from dataclasses import dataclass + +from opentelemetry._events import Event, EventLogger +from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.trace import SpanContext + +from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + +@dataclass(frozen=True, slots=True) +class GenAIEventRecorder: + event_logger: EventLogger + + def record_operation_exception( + self, + span_context: SpanContext, + error_type: str, + message: str, + stack_trace: str | None, + timestamp_ns: int | None, + ) -> None: + # ``exception.type`` and ``exception.message`` are the semconv-required + # pair and always ride the event; only the recommended stacktrace is + # conditional on the payload carrying one. + stacktrace = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else () + self.event_logger.emit( + Event( + name=GenAIEvent.OPERATION_EXCEPTION, + timestamp=timestamp_ns, + trace_id=span_context.trace_id, + span_id=span_context.span_id, + trace_flags=span_context.trace_flags, + severity_number=SeverityNumber.WARN, + attributes=dict( + ( + (ExceptionEvent.TYPE, error_type), + (ExceptionEvent.MESSAGE, message), + *stacktrace, + ) + ), + ) + ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ac971c6daa8..ced65aa1ec3 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -2,9 +2,20 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable -from opentelemetry import baggage, metrics +from opentelemetry import _logs, baggage, metrics +from opentelemetry._events import EventLogger +from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context from opentelemetry.metrics import MeterProvider, NoOpMeterProvider +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider +from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, + ConsoleLogExporter, + InMemoryLogExporter, + LogExporter, + SimpleLogRecordProcessor, +) from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider @@ -224,6 +235,112 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) +def _otlp_logs_endpoint(endpoint: str | None) -> str | None: + """Point an OTLP/HTTP base endpoint at the ``/v1/logs`` signal path. + + The OTLP/HTTP exporter only appends ``/v1/logs`` when it reads + ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used + verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint`` + for the logs signal (rewriting a sibling signal path when present). + """ + if not endpoint: + return endpoint + endpoint = endpoint.rstrip("/") + if endpoint.endswith("/v1/logs"): + return endpoint + for other_signal in ("/v1/traces", "/v1/metrics"): + if endpoint.endswith(other_signal): + return endpoint[: -len(other_signal)] + "/v1/logs" + return endpoint + "/v1/logs" + + +def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter: + """Build a log exporter mirroring the exporter selection of the other signals. + + ``console`` (and any unrecognized kind) exports to the console; ``otlp_http`` + and ``otlp_grpc`` export over OTLP with the configured endpoint/headers; + ``in_memory`` buffers for tests. Like GenAI metrics, events ride the + single-destination shorthand fields, not the multi-exporter ``exporters`` list. + """ + kind = (config.exporter or "console").lower() + if kind in ("in_memory", "inmemory", "memory"): + return InMemoryLogExporter() + if kind in ("otlp_http", "http", "http/protobuf", "http/json"): + from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter as HTTPLogExporter, + ) + + return HTTPLogExporter( + endpoint=_otlp_logs_endpoint(config.endpoint), + headers=parse_headers(config.headers), + ) + if kind in ("otlp_grpc", "grpc"): + try: + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter as GRPCLogExporter, + ) + except ImportError as exc: + raise ImportError( + "OpenTelemetry OTLP gRPC log exporter is not available. Install " + "`opentelemetry-exporter-otlp` and `grpcio` (or `litellm[grpc]`)." + ) from exc + + return GRPCLogExporter(endpoint=config.endpoint, headers=parse_headers(config.headers)) + return ConsoleLogExporter() + + +def build_logger_provider( + config: OpenTelemetryV2Config, + log_exporter: LogExporter | None = None, +) -> SDKLoggerProvider: + """Build the :class:`LoggerProvider` GenAI events export through. + + ``log_exporter`` is an explicit override (tests inject an + ``InMemoryLogExporter``); otherwise the exporter is selected from the config's + exporter kind via :func:`build_log_exporter`. Console and in-memory exporters + get a Simple processor (synchronous export, which tests rely on), everything + else a Batch processor — the same split as span processing. + """ + exporter = log_exporter if log_exporter is not None else build_log_exporter(config) + provider = SDKLoggerProvider(resource=build_resource(config)) + use_simple = isinstance(exporter, (ConsoleLogExporter, InMemoryLogExporter)) + provider.add_log_record_processor( + SimpleLogRecordProcessor(exporter) if use_simple else BatchLogRecordProcessor(exporter) + ) + return provider + + +def resolve_logger_provider( + config: OpenTelemetryV2Config, + logger_provider: SDKLoggerProvider | None = None, +) -> SDKLoggerProvider | None: + """Resolve the :class:`LoggerProvider` GenAI events record through, or ``None`` + when the operator has opted out of the logs signal. + + Same resolution order as :func:`resolve_meter_provider`: an injected provider + wins (DI/tests); an operator-configured SDK global is reused so events ride + their pipeline; an explicit ``NoOpLoggerProvider`` global is an opt-out and + yields ``None``, so no event is ever built. Only the default placeholder + global makes V2 build a provider from the config and publish it as the global. + """ + if logger_provider is not None: + return logger_provider + + existing: LoggerProvider = _logs.get_logger_provider() + if isinstance(existing, SDKLoggerProvider): + return existing + if isinstance(existing, NoOpLoggerProvider): + return None + + provider = build_logger_provider(config) + _logs.set_logger_provider(provider) + return provider + + +def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger: + return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version) + + def build_meter_provider( config: OpenTelemetryV2Config, metric_reader: "MetricReader | None" = None, diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index abc171f900a..410bb9623fe 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -3,52 +3,69 @@ Declarative fallback generalizations for unknown / newly-released models. The ``fallback_generalizations`` block in ``model_prices_and_context_window.json`` holds an ordered list of rules. Each rule pairs a single case-insensitive regex -with the metadata to apply when a model name has no exact entry in the cost map. -The metadata is a partial cost-map entry: ``litellm_provider`` drives provider -routing, and the remaining fields (``mode``, ``supports_*``, context window, -pricing, ...) drive ``get_model_info`` / ``supports_*``. +with a ``model_info`` dict, and the structure of ``model_info`` decides which of +two kinds the rule is. -Precedence: rules are evaluated in file order and the first match wins. They are -consulted only after exact and case-insensitive lookups miss, so an exact entry -always takes precedence over a rule. +A ROUTING rule carries exactly one ``model_info`` key, ``litellm_provider``. It is +consumed only by ``get_llm_provider`` bare-id inference: the first routing rule +whose regex matches decides the provider. Routing rules never contribute to model +info. + +A CAPABILITY rule carries any ``model_info`` keys except ``litellm_provider`` +(``mode``, ``supports_*``, context window, pricing, ...). It is consumed by +``get_model_info`` fallback resolution: the ``model_info`` of ALL capability rules +whose regex matches is unioned in file order, with later rules overriding earlier +ones on key conflicts, and the caller backfills ``litellm_provider`` with the +provider it requested. If no capability rule matches, model-info resolution misses +as if no rules existed. + +LEGACY-SCHEMA SHIM (temporary, until the new-schema JSON reaches main): released +proxies fetch this JSON remotely from main, whose block still ships the old schema +where a rule mixes ``litellm_provider`` with capability keys and may inherit a +parent's ``model_info`` via ``extends``. Such a legacy rule is tolerated rather +than skipped: ``extends`` is resolved once at install time (single level, against +raw parents), and the resolved rule acts as BOTH kinds, a routing rule (its +``litellm_provider`` participates in first-hit inference) and a capability rule +(its full ``model_info``, provider included, participates in the union). New-schema +rules never mix the two and never use ``extends``. A rule whose +``litellm_provider`` is not a string is invalid and is warned about and skipped +(a warning rather than a crash, for the same remote-fetch reason). + +Rules are only consulted after exact and case-insensitive lookups miss, so an +exact cost-map entry always takes precedence over any rule. Patterns are matched case-insensitively with ``re.search`` and are not implicitly -anchored: a rule must include ``^`` and ``$`` (as the shipped rules do) to bind to -the whole model name, otherwise it matches as a substring. Keeping anchoring in the -regex makes the rule the single, self-contained source of truth for what it matches. - -A rule may set ``extends`` to the ``name`` of another rule to inherit that rule's -``model_info``; the rule's own ``model_info`` overrides the inherited keys, so a -narrow rule (for example a version-gated capability flag) carries only its delta -instead of duplicating the parent's pricing block. Inheritance is resolved once, -at install time, against each rule's raw (unresolved) ``model_info``; it is a -single level (a parent that itself extends is not chained). +anchored: a rule must include ``^`` and ``$`` to bind to the whole model name, +otherwise it matches as a substring. Keeping anchoring in the regex makes the rule +the single, self-contained source of truth for what it matches. Any other keys on a rule (for example a free-text ``description`` documenting what the regex matches) are ignored by the engine and exist only for the reader. -The compiled-regex list is built once and cached. ``match_fallback_generalization`` -is O(number of rules); callers must only invoke it on a cache miss. +Rules are compiled and classified once, at install time. The match functions are +O(number of rules); callers must only invoke them on a cache miss. """ import re -from typing import Optional +from dataclasses import dataclass +from typing import Optional, Union from litellm._logging import verbose_logger NAME_FIELD = "name" PATTERN_FIELD = "pattern" MODEL_INFO_FIELD = "model_info" -EXTENDS_FIELD = "extends" +PROVIDER_KEY = "litellm_provider" +LEGACY_EXTENDS_FIELD = "extends" -def _resolve_extends(rules: list) -> list: - """Expand ``extends`` inheritance so each rule's ``model_info`` is self-contained. +def _resolve_legacy_extends(rules: list) -> list: + """Expand legacy ``extends`` inheritance so each rule's ``model_info`` is self-contained. - A rule with ``extends: `` is rewritten with ``model_info`` set to the parent's - ``model_info`` overlaid by its own. Resolution is single-level and uses each rule's - raw ``model_info`` as the parent source. Non-dict rules and dangling parents are - passed through unchanged. + Compatibility shim for the old remote schema: single level, resolved against each + parent's raw ``model_info``, with the child's own keys winning on conflict. Non-dict + rules and dangling parents pass through unchanged; new-schema rules carry no + ``extends`` and are untouched. """ base_by_name = { rule[NAME_FIELD]: rule[MODEL_INFO_FIELD] @@ -58,84 +75,138 @@ def _resolve_extends(rules: list) -> list: and isinstance(rule.get(MODEL_INFO_FIELD), dict) } - def resolved(rule: dict) -> dict: - parent_name = rule.get(EXTENDS_FIELD) + def resolved(rule: object) -> object: + if not isinstance(rule, dict): + return rule + parent_name = rule.get(LEGACY_EXTENDS_FIELD) own_info = rule.get(MODEL_INFO_FIELD) parent_info = base_by_name.get(parent_name) if isinstance(parent_name, str) else None if parent_info is None or not isinstance(own_info, dict): return rule return {**rule, MODEL_INFO_FIELD: {**parent_info, **own_info}} - return [resolved(rule) if isinstance(rule, dict) else rule for rule in rules] + return [resolved(rule) for rule in rules] + + +@dataclass(frozen=True, slots=True) +class _RoutingRule: + pattern: re.Pattern + provider: str + + +@dataclass(frozen=True, slots=True) +class _CapabilityRule: + pattern: re.Pattern + model_info: dict + + +_CompiledRule = Union[_RoutingRule, _CapabilityRule] + + +def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: + if not isinstance(rule, dict): + return () + pattern = rule.get(PATTERN_FIELD) + model_info = rule.get(MODEL_INFO_FIELD) + if not isinstance(pattern, str) or not isinstance(model_info, dict): + verbose_logger.warning( + "LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').", + rule.get(NAME_FIELD, pattern), + PATTERN_FIELD, + MODEL_INFO_FIELD, + ) + return () + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error as e: + verbose_logger.warning( + "LiteLLM: skipping fallback generalization rule with invalid regex %r: %s", + pattern, + e, + ) + return () + if PROVIDER_KEY not in model_info: + return (_CapabilityRule(pattern=compiled, model_info=model_info),) + provider = model_info[PROVIDER_KEY] + if not isinstance(provider, str): + verbose_logger.warning( + "LiteLLM: skipping invalid fallback generalization rule %s: '%s' in '%s' must be a string.", + rule.get(NAME_FIELD, pattern), + PROVIDER_KEY, + MODEL_INFO_FIELD, + ) + return () + if len(model_info) == 1: + return (_RoutingRule(pattern=compiled, provider=provider),) + return ( + _RoutingRule(pattern=compiled, provider=provider), + _CapabilityRule(pattern=compiled, model_info=model_info), + ) class _FallbackGeneralizations: - """Holds the active rule list and its lazily-compiled regex cache.""" + """Holds the raw rule list and its install-time-compiled routing and capability rules.""" def __init__(self) -> None: - self.rules: list[dict] = [] - self._compiled: Optional[list[tuple[re.Pattern, dict]]] = None + self.rules: list = [] + self.routing_rules: tuple = () + self.capability_rules: tuple = () - def set_rules(self, rules: Optional[list[dict]]) -> None: - self.rules = rules if isinstance(rules, list) else [] - self._compiled = None + def set_rules(self, rules: Optional[list]) -> None: + installed = rules if isinstance(rules, list) else [] + compiled = tuple(kind for rule in _resolve_legacy_extends(installed) for kind in _compile_rule(rule)) + self.rules = installed + self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule)) + self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule)) - def _compile(self) -> list[tuple[re.Pattern, dict]]: - compiled: list[tuple[re.Pattern, dict]] = [] - for rule in self.rules: - if not isinstance(rule, dict): - continue - pattern = rule.get(PATTERN_FIELD) - model_info = rule.get(MODEL_INFO_FIELD) - if not isinstance(pattern, str) or not isinstance(model_info, dict): - verbose_logger.warning( - "LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').", - rule.get("name", pattern), - PATTERN_FIELD, - MODEL_INFO_FIELD, - ) - continue - try: - compiled.append((re.compile(pattern, re.IGNORECASE), model_info)) - except re.error as e: - verbose_logger.warning( - "LiteLLM: skipping fallback generalization rule with invalid regex %r: %s", - pattern, - e, - ) - return compiled - - def match(self, model: str) -> Optional[dict]: + def match_routing(self, model: str) -> Optional[str]: if not model: return None - if self._compiled is None: - self._compiled = self._compile() - for pattern, model_info in self._compiled: - if pattern.search(model) is not None: - return dict(model_info) - return None + return next( + (rule.provider for rule in self.routing_rules if rule.pattern.search(model) is not None), + None, + ) + + def match_capabilities(self, model: str) -> Optional[dict]: + if not model: + return None + matched = tuple(rule.model_info for rule in self.capability_rules if rule.pattern.search(model) is not None) + if not matched: + return None + return {key: value for model_info in matched for key, value in model_info.items()} _registry = _FallbackGeneralizations() -def set_fallback_generalizations(rules: Optional[list[dict]]) -> None: - """Install the active rule list and invalidate the compiled-regex cache. +def set_fallback_generalizations(rules: Optional[list]) -> None: + """Install the active rule list, compiling and classifying each rule. - ``extends`` inheritance is resolved here, once, before the rules are stored. - Called once when the model cost map is loaded (and again on any reload). + Legacy ``extends`` inheritance is resolved here, once, before classification; + a legacy rule mixing ``litellm_provider`` with capability keys installs as both + kinds. Malformed and invalid-regex rules are warned about and skipped. Called + once when the model cost map is loaded (and again on any reload). """ - _registry.set_rules(_resolve_extends(rules) if isinstance(rules, list) else rules) + _registry.set_rules(rules) -def get_fallback_generalization_rules() -> list[dict]: +def get_fallback_generalization_rules() -> list: """Return the raw rule list (read-only view for callers/tests).""" return _registry.rules -def match_fallback_generalization(model: str) -> Optional[dict]: - """Return the ``model_info`` of the first rule whose regex matches ``model``. +def match_routing_generalization(model: str) -> Optional[str]: + """Return the provider of the first routing rule whose regex matches ``model``. O(number of rules). Only call this once exact lookups have missed. """ - return _registry.match(model) + return _registry.match_routing(model) + + +def match_capability_generalizations(model: str) -> Optional[dict]: + """Return the union of the ``model_info`` of every capability rule matching ``model``. + + Later rules override earlier ones on key conflicts. Returns ``None`` when no + capability rule matches. O(number of rules); only call once exact lookups have missed. + """ + return _registry.match_capabilities(model) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 61a73201c43..487a7b7e25f 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import litellm from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH from litellm.litellm_core_utils.fallback_generalizations import ( - match_fallback_generalization, + match_routing_generalization, ) from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.secret_managers.main import get_secret, get_secret_str @@ -346,6 +346,9 @@ def get_llm_provider( elif endpoint == "https://pinstripes.io/v1": custom_llm_provider = "pinstripes" dynamic_api_key = get_secret_str("PINSTRIPES_API_KEY") + elif endpoint == "https://api.meta.ai/v1": + custom_llm_provider = "meta" + dynamic_api_key = get_secret_str("META_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) @@ -471,12 +474,10 @@ def get_llm_provider( custom_llm_provider = "sap" # Last resort for an otherwise-unknown model: a declarative - # fallback-generalization rule (e.g. routes future claude-* to anthropic). + # fallback-generalization routing rule (e.g. routes future claude-* to anthropic). # Exact provider matches above always win; this only runs on a miss. if not custom_llm_provider: - generalization = match_fallback_generalization(model) - if generalization is not None: - custom_llm_provider = generalization.get("litellm_provider") or None + custom_llm_provider = match_routing_generalization(model) if not custom_llm_provider: if litellm.suppress_debug_info is False: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9721b797584..6033e54fb77 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -266,6 +266,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def custom_llm_provider(self) -> Optional[str]: return "anthropic" + @property + def _resolved_provider(self) -> str: + return self.custom_llm_provider or "anthropic" + @classmethod def get_config(cls, *, model: Optional[str] = None): config = super().get_config() @@ -335,23 +339,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return any(v in model_lower for v in ("opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7")) @staticmethod - def _supports_effort_level(model: str, level: str) -> bool: + def _supports_effort_level(model: str, level: str, custom_llm_provider: str) -> bool: """Check ``supports_{level}_reasoning_effort`` in the model map.""" - return AnthropicConfig._supports_model_capability(model, f"supports_{level}_reasoning_effort") + return AnthropicConfig._supports_model_capability( + model, f"supports_{level}_reasoning_effort", custom_llm_provider + ) @staticmethod - def _validate_effort_for_model(model: str, effort: Optional[str]) -> Optional[str]: + def _validate_effort_for_model(model: str, effort: Optional[str], custom_llm_provider: str) -> Optional[str]: """Return ``None`` if ``effort`` is allowed on ``model``, else an error message.""" if effort == "max" and not ( - AnthropicConfig._is_adaptive_thinking_model(model) or AnthropicConfig._supports_effort_level(model, "max") + AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider) + or AnthropicConfig._supports_effort_level(model, "max", custom_llm_provider) ): return f"effort='max' is not supported by this model. Got model: {model}" - if effort == "xhigh" and not AnthropicConfig._supports_effort_level(model, "xhigh"): + if effort == "xhigh" and not AnthropicConfig._supports_effort_level(model, "xhigh", custom_llm_provider): return f"effort='xhigh' is not supported by this model. Got model: {model}" return None @staticmethod - def _model_supports_effort_param(model: str) -> bool: + def _model_supports_effort_param(model: str, custom_llm_provider: str) -> bool: """Whether the model accepts ``output_config.effort`` at all. A model qualifies if its map entry advertises ``supports_output_config`` @@ -359,10 +366,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): signals: e.g. Claude Opus 4.5 supports ``output_config`` without advertising a non-default (max/xhigh) effort level. """ - if AnthropicConfig._supports_model_capability(model, "supports_output_config"): + if AnthropicConfig._supports_model_capability(model, "supports_output_config", custom_llm_provider): return True return any( - AnthropicConfig._supports_effort_level(model, level) + AnthropicConfig._supports_effort_level(model, level, custom_llm_provider) for level in ("low", "minimal", "medium", "high", "xhigh", "max") ) @@ -451,7 +458,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if ( "claude-3-7-sonnet" in model - or AnthropicConfig._is_adaptive_thinking_model(model) + or AnthropicConfig._is_adaptive_thinking_model(model, self._resolved_provider) or supports_reasoning( model=model, custom_llm_provider=self.custom_llm_provider, @@ -1159,11 +1166,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def _map_reasoning_effort( reasoning_effort: Optional[Union[REASONING_EFFORT, str]], model: str, + custom_llm_provider: str, llm_provider: str = "anthropic", ) -> Optional[AnthropicThinkingParam]: + """Capability probes read the cost map under ``custom_llm_provider``; ``llm_provider`` only tags raised exceptions.""" if reasoning_effort is None or reasoning_effort == "none": return None - if AnthropicConfig._is_adaptive_thinking_model(model): + if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider): return AnthropicThinkingParam( type="adaptive", ) @@ -1471,20 +1480,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mapped_thinking = AnthropicConfig._map_reasoning_effort( reasoning_effort=effort_value, model=model, - llm_provider=self.custom_llm_provider or "anthropic", + custom_llm_provider=self._resolved_provider, + llm_provider=self._resolved_provider, ) if mapped_thinking is None: optional_params.pop("thinking", None) optional_params.pop("output_config", None) else: optional_params["thinking"] = mapped_thinking - if AnthropicConfig._is_adaptive_thinking_model(model): + if AnthropicConfig._is_adaptive_thinking_model(model, self._resolved_provider): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(effort_value) if mapped_effort is None: AnthropicConfig._raise_invalid_reasoning_effort( model=model, value=effort_value, - llm_provider=self.custom_llm_provider or "anthropic", + llm_provider=self._resolved_provider, ) optional_params["output_config"] = {"effort": mapped_effort} elif param == "web_search_options" and isinstance(value, dict): @@ -1813,7 +1823,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_messages = anthropic_messages_pt( model=model, messages=messages, - llm_provider=self.custom_llm_provider or "anthropic", + llm_provider=self._resolved_provider, ) except Exception as e: raise AnthropicError( @@ -1902,7 +1912,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): output_config = optional_params.get("output_config") if not output_config or not isinstance(output_config, dict): return - if litellm.drop_params is True and not self._model_supports_effort_param(model): + if litellm.drop_params is True and not self._model_supports_effort_param(model, self._resolved_provider): litellm.verbose_logger.warning( DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, model, @@ -1916,14 +1926,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): raise litellm.exceptions.BadRequestError( message=(f"Invalid effort value: {effort!r}. Must be one of: 'high', 'medium', 'low', 'xhigh', 'max'"), model=model, - llm_provider=self.custom_llm_provider or "anthropic", + llm_provider=self._resolved_provider, ) - gate_error = self._validate_effort_for_model(model, effort) + gate_error = self._validate_effort_for_model(model, effort, self._resolved_provider) if gate_error is not None: raise litellm.exceptions.BadRequestError( message=gate_error, model=model, - llm_provider=self.custom_llm_provider or "anthropic", + llm_provider=self._resolved_provider, ) data["output_config"] = output_config diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index db540e5441d..89fd12a2241 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -352,18 +352,43 @@ class AnthropicModelInfo(BaseLLMModelInfo): return value if isinstance(value, bool) else None @staticmethod - def _supports_model_capability(model: str, key: str) -> bool: - """Check a boolean capability ``key`` in the model map. + def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> Optional[bool]: + """Resolve boolean capability ``key`` for ``model`` under the caller's provider. - Strips bedrock/vertex prefixes so a provider-routed Claude still - resolves to the Anthropic model-map entry. + Returns the flag when the provider-aware lookup resolves ``model`` to an + entry (or fallback rule) that sets it explicitly, and ``None`` when the + model does not resolve under that provider or the resolved entry has no + opinion on ``key``. + """ + from litellm.utils import _get_model_info_helper + + try: + resolved_model, resolved_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider + ) + value = _get_model_info_helper(model=resolved_model, custom_llm_provider=resolved_provider).get(key) + except Exception: # noqa: BLE001 # _get_model_info_helper raises bare Exception for unmapped models + return None + return value if isinstance(value, bool) else None + + @staticmethod + def _supports_model_capability(model: str, key: str, custom_llm_provider: str) -> bool: + """Check a boolean capability ``key`` in the model map under the caller's provider. + + The provider-aware lookup is authoritative when it resolves an explicit flag, + so ``key: false`` on the provider-namespaced entry wins over every fallback. + Otherwise ``_supports_factory``'s provider-level fallbacks and the raw + model-map walk remain as backstops for alias forms the lookup misses. """ from litellm.utils import _supports_factory + resolved = AnthropicModelInfo._get_provider_resolved_capability(model, key, custom_llm_provider) + if resolved is not None: + return resolved try: if _supports_factory( model=model, - custom_llm_provider="anthropic", + custom_llm_provider=custom_llm_provider, key=key, ): return True @@ -372,17 +397,24 @@ class AnthropicModelInfo(BaseLLMModelInfo): return AnthropicModelInfo._get_model_capability(model, key) is True @staticmethod - def _is_adaptive_thinking_model(model: str) -> bool: + def _is_adaptive_thinking_model(model: str, custom_llm_provider: str) -> bool: """Whether ``model`` uses adaptive thinking (``output_config.effort``). The model cost map is authoritative: an explicit ``supports_adaptive_thinking`` - entry, or a ``fallback_generalizations`` rule for unknown Claude models. The - version gate (>= 4.6, including provider-prefixed Bedrock/Vertex ids that map to - no exact entry) lives entirely in that declarative rule, not here. + entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations`` + rule for unknown Claude models. The version gate (>= 4.6, including + provider-prefixed Bedrock/Vertex ids that map to no exact entry) lives entirely + in that declarative rule, not here. """ - return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking") + return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking", custom_llm_provider) - def is_effort_used(self, optional_params: Optional[dict], model: Optional[str] = None) -> bool: + def is_effort_used( + self, + optional_params: Optional[dict], + model: Optional[str] = None, + *, + custom_llm_provider: str, + ) -> bool: """ Check if effort parameter is being used and requires a beta header. @@ -394,7 +426,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False # Claude 4.6+ models use output_config as a stable API feature — no beta header needed - if model and self._is_adaptive_thinking_model(model): + if model and self._is_adaptive_thinking_model(model, custom_llm_provider): return False # Check if reasoning_effort is provided for Claude Opus 4.5 @@ -475,6 +507,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): prompt_caching_set: bool = False, file_id_used: bool = False, mcp_server_used: bool = False, + *, + custom_llm_provider: str, ) -> List[str]: """ Get list of common beta headers based on the features that are active. @@ -487,7 +521,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas = [] # Detect features - effort_used = self.is_effort_used(optional_params, model) + effort_used = self.is_effort_used(optional_params, model, custom_llm_provider=custom_llm_provider) if effort_used: betas.append(ANTHROPIC_EFFORT_BETA_HEADER) # effort-2025-11-24 @@ -643,7 +677,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): tool_search_used = self.is_tool_search_used(tools=tools) programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools) input_examples_used = self.is_input_examples_used(tools=tools) - effort_used = self.is_effort_used(optional_params=optional_params, model=model) + effort_used = self.is_effort_used(optional_params=optional_params, model=model, custom_llm_provider="anthropic") code_execution_tool_used = self.is_code_execution_tool_used(tools=tools) container_with_skills_used = self.is_container_with_skills_used(optional_params=optional_params) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index e78802a1587..39713e0f003 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -3,6 +3,7 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple import httpx from litellm.constants import ( + ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, @@ -32,8 +33,22 @@ from ...common_utils import ( DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01" +DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING = ( + "Dropping adaptive `thinking`/`output_config.effort` for model=%s: the model " + "does not support extended thinking, or max_tokens is too small to fit the " + "minimum thinking budget." +) + class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): + @property + def custom_llm_provider(self) -> Optional[str]: + return "anthropic" + + @property + def _resolved_provider(self) -> str: + return self.custom_llm_provider or "anthropic" + def get_supported_anthropic_messages_params(self, model: str) -> list: return [ "messages", @@ -174,7 +189,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return headers, api_base @staticmethod - def _translate_reasoning_effort_to_anthropic(model: str, optional_params: Dict) -> None: + def _translate_reasoning_effort_to_anthropic(model: str, optional_params: Dict, custom_llm_provider: str) -> None: """Map OpenAI-style ``reasoning_effort`` to native Anthropic params. Caller-supplied ``thinking`` / ``output_config`` win over the alias. @@ -191,7 +206,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return try: - mapped_thinking = AnthropicConfig._map_reasoning_effort(reasoning_effort=reasoning_effort, model=model) + mapped_thinking = AnthropicConfig._map_reasoning_effort( + reasoning_effort=reasoning_effort, + model=model, + custom_llm_provider=custom_llm_provider, + ) except _BadRequestError as e: raise AnthropicError(message=str(e.message), status_code=400) @@ -201,7 +220,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return optional_params.setdefault("thinking", mapped_thinking) - if AnthropicModelInfo._is_adaptive_thinking_model(model): + if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort) if mapped_effort is None: raise AnthropicError( @@ -212,7 +231,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ), status_code=400, ) - gate_error = AnthropicConfig._validate_effort_for_model(model, mapped_effort) + gate_error = AnthropicConfig._validate_effort_for_model(model, mapped_effort, custom_llm_provider) if gate_error is not None: raise AnthropicError(message=gate_error, status_code=400) existing_output_config = optional_params.get("output_config") @@ -222,13 +241,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): optional_params["output_config"] = existing_output_config @staticmethod - def _translate_legacy_thinking_for_adaptive_model(model: str, optional_params: Dict) -> None: + def _translate_legacy_thinking_for_adaptive_model( + model: str, optional_params: Dict, custom_llm_provider: str + ) -> None: """Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7. Caller-provided ``output_config.effort`` is never overridden. """ from litellm.llms.anthropic.chat.transformation import AnthropicConfig - if not AnthropicModelInfo._is_adaptive_thinking_model(model): + if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): return thinking = optional_params.get("thinking") if not isinstance(thinking, dict) or thinking.get("type") != "enabled": @@ -236,7 +257,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): budget = int(thinking.get("budget_tokens") or 0) if budget >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and ( - AnthropicConfig._supports_effort_level(model, "xhigh") + AnthropicConfig._supports_effort_level(model, "xhigh", custom_llm_provider) ): effort = "xhigh" elif budget >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET: @@ -253,6 +274,118 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): existing_output_config.setdefault("effort", effort) optional_params["output_config"] = existing_output_config + @staticmethod + def _translate_adaptive_effort_for_non_adaptive_model( + model: str, optional_params: Dict, max_tokens: Optional[int], custom_llm_provider: str + ) -> None: + """Translate the 4.6+ adaptive-thinking interface (``thinking.type=adaptive`` + and/or ``output_config.effort``) down to what an older Anthropic model + supports. Clients like Claude Code send this interface unconditionally, so + without translation it reaches a pre-4.6 model and Anthropic rejects it with + "This model does not support the effort parameter". + + The reshape is silent, matching how the messages path already strips + unsupported ``output_config`` for older models (bedrock invoke, issue + #22797): the goal is to keep the request working, not to fail it. + + ``thinking.type=adaptive`` and ``output_config.effort`` are independent + capabilities. Adaptive thinking needs ``supports_adaptive_thinking`` (4.6+); + ``output_config.effort`` needs ``supports_output_config``, which some + non-adaptive models (e.g. Claude Opus 4.5) advertise on its own. So the two + are handled separately: + + - Adaptive-thinking models (4.6+): both are native, left untouched. + - ``supports_output_config`` but non-adaptive (Opus 4.5): keep + ``output_config.effort`` (native), only drop the unsupported adaptive + ``thinking`` block. When adaptive thinking is being dropped and the + effort level itself isn't supported by the model (e.g. ``xhigh``/``max`` + on Opus 4.5, which only accepts low/medium/high, while ``xhigh`` is + Claude Code's default), fall through to the legacy translation below + instead of forwarding a level Anthropic would reject. Effort-only + requests are always left untouched: provider subclasses own their level + normalization (bedrock clamps ``xhigh`` to the model's ceiling after + this base transform runs). + - Thinking-capable but neither (``supports_reasoning``, e.g. Haiku/Sonnet + 4.5): map effort to legacy ``thinking={type: enabled, budget_tokens}`` via + ``AnthropicConfig._map_reasoning_effort``, capped below ``max_tokens`` + (Anthropic requires ``max_tokens > budget_tokens``) and dropped when + ``max_tokens`` can't fit even the minimum budget. + - No reasoning support: ``thinking`` is dropped. + + For the last two, only the consumed ``effort`` key is removed from + ``output_config``; any residual (e.g. ``format``) is left for provider + subclasses to handle. + """ + from litellm.exceptions import BadRequestError as _BadRequestError + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider): + return + + output_config = optional_params.get("output_config") + thinking = optional_params.get("thinking") + effort = output_config.get("effort") if isinstance(output_config, dict) else None + adaptive_thinking = isinstance(thinking, dict) and thinking.get("type") == "adaptive" + if effort is None and not adaptive_thinking: + return + + if AnthropicConfig._model_supports_effort_param(model, custom_llm_provider) and ( + not adaptive_thinking + or AnthropicConfig._validate_effort_for_model(model, effort, custom_llm_provider) is None + ): + if adaptive_thinking: + optional_params.pop("thinking", None) + return + + supports_thinking = AnthropicModelInfo._supports_model_capability( + model, "supports_reasoning", custom_llm_provider + ) + try: + legacy_thinking = ( + AnthropicConfig._map_reasoning_effort( + reasoning_effort=effort or "medium", + model=model, + custom_llm_provider=custom_llm_provider, + ) + if supports_thinking + else None + ) + except _BadRequestError as e: + raise AnthropicError(message=str(e.message), status_code=400) + capped_thinking = ( + AnthropicMessagesConfig._cap_thinking_budget_to_max_tokens(legacy_thinking, max_tokens) + if legacy_thinking is not None + else None + ) + + if capped_thinking is not None: + optional_params["thinking"] = capped_thinking + else: + verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_EFFORT_WARNING, model) + optional_params.pop("thinking", None) + + if isinstance(output_config, dict) and "effort" in output_config: + residual = {k: v for k, v in output_config.items() if k != "effort"} + if residual: + optional_params["output_config"] = residual + else: + optional_params.pop("output_config", None) + + @staticmethod + def _cap_thinking_budget_to_max_tokens(thinking: Dict, max_tokens: Optional[int]) -> Optional[Dict]: + """Cap a legacy ``thinking.budget_tokens`` below ``max_tokens`` (Anthropic + requires ``max_tokens > budget_tokens``). Returns the (possibly capped) + thinking dict, or ``None`` when ``max_tokens`` is too small to fit even the + minimum thinking budget and thinking should be dropped.""" + budget = thinking.get("budget_tokens") + if max_tokens is None or not isinstance(budget, int): + return thinking + if max_tokens <= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: + return None + if budget < max_tokens: + return thinking + return {**thinking, "budget_tokens": max_tokens - 1} + def transform_anthropic_messages_request( self, model: str, @@ -277,11 +410,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): self._translate_reasoning_effort_to_anthropic( model=model, optional_params=anthropic_messages_optional_request_params, + custom_llm_provider=self._resolved_provider, ) self._translate_legacy_thinking_for_adaptive_model( model=model, optional_params=anthropic_messages_optional_request_params, + custom_llm_provider=self._resolved_provider, + ) + + self._translate_adaptive_effort_for_non_adaptive_model( + model=model, + optional_params=anthropic_messages_optional_request_params, + max_tokens=max_tokens, + custom_llm_provider=self._resolved_provider, ) system_param = anthropic_messages_optional_request_params.get("system") diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 1de18701a2f..8cee35989af 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -21,6 +21,10 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): and Azure endpoint format. """ + @property + def custom_llm_provider(self) -> Optional[str]: + return "azure_ai" + def should_strip_billing_metadata(self) -> bool: return True diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5a8ada45651..be904fb27be 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -423,6 +423,7 @@ class AmazonConverseConfig(BaseConfig): mapped_thinking = AnthropicConfig._map_reasoning_effort( reasoning_effort=reasoning_effort, model=model, + custom_llm_provider="bedrock", llm_provider="bedrock_converse", ) if mapped_thinking is None: @@ -430,7 +431,7 @@ class AmazonConverseConfig(BaseConfig): optional_params.pop("output_config", None) else: optional_params["thinking"] = mapped_thinking - if AnthropicConfig._is_adaptive_thinking_model(model): + if AnthropicConfig._is_adaptive_thinking_model(model, "bedrock"): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort) if mapped_effort is None: AnthropicConfig._raise_invalid_reasoning_effort( @@ -465,7 +466,7 @@ class AmazonConverseConfig(BaseConfig): model=model, llm_provider="bedrock_converse", ) - error = AnthropicConfig._validate_effort_for_model(model=model, effort=effort) + error = AnthropicConfig._validate_effort_for_model(model=model, effort=effort, custom_llm_provider="bedrock") if error is not None: raise litellm.exceptions.BadRequestError( message=error, @@ -1279,7 +1280,7 @@ class AmazonConverseConfig(BaseConfig): if anthropic_output_config is not None and isinstance(anthropic_output_config, dict): if base_model.startswith("anthropic"): - if litellm.drop_params is True and not AnthropicConfig._model_supports_effort_param(model): + if litellm.drop_params is True and not AnthropicConfig._model_supports_effort_param(model, "bedrock"): litellm.verbose_logger.warning( DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, model, @@ -1422,7 +1423,7 @@ class AmazonConverseConfig(BaseConfig): if ( isinstance(output_config, dict) and output_config.get("effort") is not None - and not AnthropicConfig._is_adaptive_thinking_model(model) + and not AnthropicConfig._is_adaptive_thinking_model(model, "bedrock") ): from litellm.types.llms.anthropic import ( ANTHROPIC_EFFORT_BETA_HEADER, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 60d532eb8c5..6b5cb304bec 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -115,7 +115,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): keeps working. Non-adaptive models and models without a ceiling are left untouched. """ - if not AnthropicConfig._is_adaptive_thinking_model(model): + if not AnthropicConfig._is_adaptive_thinking_model(model, "bedrock"): return effort = params.get("reasoning_effort") if not isinstance(effort, str): @@ -228,7 +228,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): custom_llm_provider="bedrock", key="supports_output_config", ) - or AnthropicConfig._model_supports_effort_param(model) + or AnthropicConfig._model_supports_effort_param(model, "bedrock") ): if anthropic_request.pop("output_config", None) is not None: verbose_logger.warning( @@ -269,6 +269,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): prompt_caching_set=False, file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), + custom_llm_provider="bedrock", ) beta_set.update(auto_betas) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index 0868d9bddfe..6f5ccececc7 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -54,7 +54,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): tool_search_used=self.is_tool_search_used(tools=optional_params.get("tools")), programmatic_tool_calling_used=self.is_programmatic_tool_calling_used(tools=optional_params.get("tools")), input_examples_used=self.is_input_examples_used(tools=optional_params.get("tools")), - effort_used=self.is_effort_used(optional_params=optional_params, model=model), + effort_used=self.is_effort_used( + optional_params=optional_params, model=model, custom_llm_provider="anthropic" + ), user_anthropic_beta_headers=self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ), diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index f5309d521a9..3ef7cba8220 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -77,6 +77,10 @@ class AmazonAnthropicClaudeMessagesConfig( DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31" + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()) def __init__(self, **kwargs): @@ -93,26 +97,48 @@ class AmazonAnthropicClaudeMessagesConfig( return [{"type": "text", "text": value}] return [value] - def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict) -> None: - """Bedrock Invoke rejects ``role: "system"`` entries inside ``messages`` on - some Claude aliases; Anthropic Messages carries that content in the - top-level ``system`` field. Move any such entries into ``system`` before - the Invoke request is built.""" + @staticmethod + def _is_system_role_message(message: Any) -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict, model: str) -> None: + """Bedrock Invoke validates ``role: "system"`` entries inside ``messages`` + per model. Models carrying ``supports_mid_conversation_system`` in the + cost map (the Opus 4.8 family) only reject a leading run ("messages.0: + use the top-level 'system' parameter for the initial system prompt") and + accept mid-conversation entries (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) in place, where they + MUST stay: hoisting one mutates the ``system`` prefix and invalidates the + prompt cache for the entire message history. Older Claude models (Opus + 4.7, Sonnet 4.6, Haiku 4.5, ...) reject the role in every position + ("role 'system' is not supported on this model"), so without the flag + every system entry is hoisted into the top-level ``system`` field. + Billing-header system blocks are stripped from the top-level ``system`` + field regardless of whether anything was hoisted.""" messages = anthropic_messages_request.get("messages") if not isinstance(messages, list): return - system_role_messages = [m for m in messages if isinstance(m, dict) and m.get("role") == "system"] - if not system_role_messages: - return - - anthropic_messages_request["messages"] = [ - m for m in messages if not (isinstance(m, dict) and m.get("role") == "system") - ] + if _supports_factory( + model=model, + custom_llm_provider="bedrock", + key="supports_mid_conversation_system", + ): + leading_count = next( + (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), + len(messages), + ) + hoisted = messages[:leading_count] + remaining = messages[leading_count:] + else: + hoisted = [m for m in messages if self._is_system_role_message(m)] + remaining = [m for m in messages if not self._is_system_role_message(m)] + if hoisted: + anthropic_messages_request["messages"] = remaining system_content = [ block for source in ( anthropic_messages_request.get("system"), - *(m.get("content") for m in system_role_messages), + *(m.get("content") for m in hoisted), ) for block in self._as_system_content_blocks(source) ] @@ -247,7 +273,7 @@ class AmazonAnthropicClaudeMessagesConfig( Returns: True if the model supports extended thinking on Bedrock """ - if AnthropicModelInfo._is_adaptive_thinking_model(model): + if AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock"): return True model_lower = model.lower() @@ -297,7 +323,7 @@ class AmazonAnthropicClaudeMessagesConfig( if not self._supports_extended_thinking_on_bedrock(model): return False - is_adaptive_thinking_model = AnthropicModelInfo._is_adaptive_thinking_model(model) + is_adaptive_thinking_model = AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock") thinking = anthropic_messages_request.get("thinking") if isinstance(thinking, dict): @@ -553,6 +579,7 @@ class AmazonAnthropicClaudeMessagesConfig( mcp_server_used=anthropic_model_info.is_mcp_server_used( anthropic_messages_optional_request_params.get("mcp_servers") ), + custom_llm_provider="bedrock", ) beta_set.update(auto_betas) @@ -619,7 +646,7 @@ class AmazonAnthropicClaudeMessagesConfig( path degrades ``xhigh`` -> ``max`` rather than 400-ing. Non-adaptive models and models without a ceiling are left untouched. """ - if not AnthropicModelInfo._is_adaptive_thinking_model(model): + if not AnthropicModelInfo._is_adaptive_thinking_model(model, "bedrock"): return effort = optional_params.get("reasoning_effort") if not isinstance(effort, str): @@ -648,7 +675,7 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages_for_bedrock(anthropic_messages_request) + self._normalize_system_role_messages_for_bedrock(anthropic_messages_request, model=model) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### @@ -707,7 +734,7 @@ class AmazonAnthropicClaudeMessagesConfig( custom_llm_provider="bedrock", key="supports_output_config", ) - or AnthropicConfig._model_supports_effort_param(model) + or AnthropicConfig._model_supports_effort_param(model, "bedrock") ): if anthropic_messages_request.pop("output_config", None) is not None: verbose_logger.warning( @@ -744,7 +771,7 @@ class AmazonAnthropicClaudeMessagesConfig( if ( litellm.drop_params is True and "output_config" in anthropic_messages_request - and not AnthropicConfig._model_supports_effort_param(model) + and not AnthropicConfig._model_supports_effort_param(model, "bedrock") ): verbose_logger.warning( DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index ba8c312ea51..9c05899c719 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -181,6 +181,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + @property + def custom_llm_provider(self) -> Optional[str]: + return "databricks" + @classmethod def get_config(cls): return super().get_config() @@ -372,6 +376,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): mapped_thinking = AnthropicConfig._map_reasoning_effort( reasoning_effort=reasoning_effort_value, model=model, + custom_llm_provider="databricks", llm_provider="databricks", ) if mapped_thinking is None: @@ -379,7 +384,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): optional_params.pop("output_config", None) else: optional_params["thinking"] = mapped_thinking - if AnthropicConfig._is_adaptive_thinking_model(model): + if AnthropicConfig._is_adaptive_thinking_model(model, "databricks"): mapped_effort: Optional[str] = None if isinstance(reasoning_effort_value, str): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort_value) diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index fb3f0a4e159..4d7b003c48f 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -25,6 +25,10 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig): super().__init__() self.authenticator = Authenticator() + @property + def custom_llm_provider(self) -> Optional[str]: + return "github_copilot" + def handles_web_search_natively(self) -> bool: """ Copilot's /v1/messages endpoint does not execute ``web_search`` tools, so diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 3c763ed9b9b..31c913d5d4e 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -91,7 +91,7 @@ def create_config_class(provider: SimpleProviderConfig): def get_supported_openai_params(self, model: str) -> list: """Get supported OpenAI params, excluding tool-related params for models that don't support function calling.""" - from litellm.utils import supports_function_calling + from litellm.utils import supports_function_calling, supports_reasoning supported_params = super().get_supported_openai_params(model=model) @@ -113,6 +113,10 @@ def create_config_class(provider: SimpleProviderConfig): f"function calling — removed tool-related params from supported params." ) + _supports_reasoning = supports_reasoning(model=model, custom_llm_provider=provider.slug) + if _supports_reasoning and "reasoning_effort" not in supported_params: + supported_params.append("reasoning_effort") + return supported_params def map_openai_params( diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 0df8c6e830b..0d593d8d0f4 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,8 +1,11 @@ from typing import Any, Optional +import litellm from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) +from litellm.llms.openai_like.json_loader import SimpleProviderConfig +from litellm.secret_managers.main import get_secret_str DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01" @@ -67,3 +70,69 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): if base.endswith("/v1"): base = base[: -len("/v1")] return f"{base}/v1/messages" + + +class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): + """ + Provider-level native Anthropic Messages passthrough for JSON-configured + OpenAI-compatible providers whose ``supported_endpoints`` in providers.json + includes ``"/v1/messages"``. Resolves the api key and api base from the + provider's configured env vars, then forwards the Anthropic payload + untranslated like ``OpenAILikeAnthropicMessagesConfig``. + """ + + def __init__(self, provider: SimpleProviderConfig): + super().__init__() + self._provider = provider + + @property + def custom_llm_provider(self) -> Optional[str]: + return self._provider.slug + + def should_strip_billing_metadata(self) -> bool: + return True + + def _resolve_api_key(self, api_key: Optional[str]) -> Optional[str]: + return api_key or get_secret_str(self._provider.api_key_env) or litellm.api_key + + def _resolve_api_base(self, api_base: Optional[str]) -> str: + env_api_base = get_secret_str(self._provider.api_base_env) if self._provider.api_base_env else None + return api_base or env_api_base or self._provider.base_url + + def validate_anthropic_messages_environment( + self, + headers: dict[str, str], + model: str, + messages: list[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> tuple[dict[str, str], Optional[str]]: + return super().validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=self._resolve_api_key(api_key), + api_base=api_base, + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + return super().get_complete_url( + api_base=self._resolve_api_base(api_base), + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + stream=stream, + ) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index d87346fea70..164100d4194 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -168,6 +168,13 @@ }, "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] }, + "meta": { + "base_url": "https://api.meta.ai/v1", + "api_key_env": "META_API_KEY", + "api_base_env": "META_API_BASE", + "base_class": "openai_gpt", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + }, "pinstripes": { "base_url": "https://pinstripes.io/v1", "api_key_env": "PINSTRIPES_API_KEY", diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 8566496bf9c..de72795cabc 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -17,6 +17,10 @@ from ..output_params_utils import sanitize_vertex_anthropic_output_params class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase): + @property + def custom_llm_provider(self) -> Optional[str]: + return "vertex_ai" + def should_strip_billing_metadata(self) -> bool: return True diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py index 280cc1c888a..b87d05ab1fd 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -26,7 +26,7 @@ def _model_accepts_output_config_effort(model: str) -> bool: """ from litellm.llms.anthropic.chat.transformation import AnthropicConfig - return AnthropicConfig._model_supports_effort_param(model) + return AnthropicConfig._model_supports_effort_param(model, "vertex_ai") def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index c8d91be359b..8fcefb04b34 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -112,6 +112,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): prompt_caching_set=self.is_cache_control_set(messages), file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), + custom_llm_provider="vertex_ai", ) beta_set = set(auto_betas) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 26d54c05ffb..46cb033b203 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1359,6 +1359,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1393,6 +1394,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1427,6 +1429,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1461,6 +1464,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1481,6 +1485,7 @@ "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1516,6 +1521,7 @@ "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1551,6 +1557,7 @@ "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1586,6 +1593,7 @@ "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1621,6 +1629,7 @@ "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1703,6 +1712,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1737,6 +1747,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1771,6 +1782,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1805,6 +1817,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1839,6 +1852,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1873,6 +1887,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -24917,6 +24932,42 @@ "supports_function_calling": true, "supports_tool_choice": false }, + "meta/muse-spark-1.1": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "meta", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.25e-06, + "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "meta_llama/Llama-3.3-70B-Instruct": { "litellm_provider": "meta_llama", "max_input_tokens": 128000, @@ -43808,20 +43859,26 @@ "fallback_generalizations": { "rules": [ { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", + "name": "bedrock-claude-ids", + "pattern": "anthropic\\.claude-", + "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.", "model_info": { - "supports_adaptive_thinking": true + "litellm_provider": "bedrock" } }, { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "name": "anthropic-claude-ids", + "pattern": "^claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?$", + "description": "A bare Claude family-major id with an optional minor and an optional 8-digit date suffix, anchored to the whole name, so claude-newfamily-5 routes like claude-newfamily-5-1 does. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.", + "model_info": { + "litellm_provider": "anthropic" + } + }, + { + "name": "claude-family-baseline", + "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", "model_info": { - "litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, @@ -43837,6 +43894,22 @@ "supports_pdf_input": true, "supports_system_messages": true } + }, + { + "name": "claude-adaptive-thinking", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "claude-mid-conversation-system", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "model_info": { + "supports_mid_conversation_system": true + } } ] } diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 1d31e33c77c..6cdf49818e6 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -32,6 +32,9 @@ def is_text_content_call_type(call_type: str) -> bool: return call_type in TEXT_CONTENT_CALL_TYPES +TEXT_PART_TYPES: FrozenSet[str] = frozenset({"text", "input_text", "output_text"}) + + def _iter_text_parts_in_content(content: Any) -> Iterator[str]: """Yield text fragments from a ``message.content`` value (string or multimodal list). Non-text parts (images, audio, …) are skipped.""" @@ -48,7 +51,7 @@ def _iter_text_parts_in_content(content: Any) -> Iterator[str]: continue if not isinstance(part, dict): continue - if part.get("type") == "text": + if part.get("type") in TEXT_PART_TYPES: text = part.get("text") if isinstance(text, str) and text: yield text @@ -58,14 +61,20 @@ def _coerce_input_to_messages(input_value: Any) -> List[Dict[str, Any]]: """Coerce a Responses-API ``data["input"]`` value into chat-style messages.""" if isinstance(input_value, str): return [{"role": "user", "content": input_value}] - if isinstance(input_value, list): - if input_value and all(isinstance(item, dict) and "role" in item for item in input_value): - return list(input_value) - # Mixed lists (content-part dicts + bare strings) and pure - # string/dict lists all become a single user message; the content - # iterator below handles each element type uniformly. - return [{"role": "user", "content": input_value}] - return [] + if not isinstance(input_value, list): + return [] + messages: List[Dict[str, Any]] = [] + for item in input_value: + if isinstance(item, str): + messages.append({"role": "user", "content": item}) + elif isinstance(item, dict): + if item.get("type") in TEXT_PART_TYPES: + messages.append({"role": item.get("role") or "user", "content": [item]}) + elif "content" in item: + messages.append({"role": item.get("role") or "user", "content": item["content"]}) + elif item.get("type") == "function_call_output" and "output" in item: + messages.append({"role": item.get("role") or "tool", "content": item["output"]}) + return messages def _iter_inspection_messages(data: Dict[str, Any]) -> Iterator[Dict[str, Any]]: @@ -112,7 +121,7 @@ def walk_user_text(data: Dict[str, Any], visit: Callable[[str], str]) -> int: new_parts.append(visit(part)) elif ( isinstance(part, dict) - and part.get("type") == "text" + and part.get("type") in TEXT_PART_TYPES and isinstance(part.get("text"), str) and part["text"] ): @@ -136,25 +145,20 @@ def walk_user_text(data: Dict[str, Any], visit: Callable[[str], str]) -> int: data["input"] = visit(input_value) return visited if isinstance(input_value, list): - # List of full messages: rewrite each message's content. - if input_value and all(isinstance(item, dict) and "role" in item for item in input_value): - for item in input_value: - if "content" in item: - item["content"] = _rewrite_content(item["content"]) - return visited - # List of content parts and/or bare strings: rewrite in place. for idx, item in enumerate(input_value): - if isinstance(item, str) and item: - visited += 1 - input_value[idx] = visit(item) - elif ( - isinstance(item, dict) - and item.get("type") == "text" - and isinstance(item.get("text"), str) - and item["text"] - ): - visited += 1 - input_value[idx] = {**item, "text": visit(item["text"])} + if isinstance(item, str): + if item: + visited += 1 + input_value[idx] = visit(item) + elif isinstance(item, dict): + if item.get("type") in TEXT_PART_TYPES: + if isinstance(item.get("text"), str) and item["text"]: + visited += 1 + input_value[idx] = {**item, "text": visit(item["text"])} + elif "content" in item: + item["content"] = _rewrite_content(item["content"]) + elif item.get("type") == "function_call_output" and "output" in item: + item["output"] = _rewrite_content(item["output"]) return visited return visited diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index e7d9406ae3b..d22243cbe88 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -93,11 +93,10 @@ class AimGuardrail(CustomGuardrail): user_email=user_email, litellm_call_id=call_id, ) - # Covers multimodal list content + Responses-API input. response = await self.async_handler.post( f"{self.api_base}/fw/v1/analyze", headers=headers, - json={"messages": build_inspection_messages(data)}, + json={"messages": self._build_aim_inspection_messages(data)}, ) response.raise_for_status() res = response.json() @@ -116,6 +115,15 @@ class AimGuardrail(CustomGuardrail): verbose_proxy_logger.error(f"Aim: {action_type} action") return data + @staticmethod + def _build_aim_inspection_messages(data: dict) -> list[dict[str, str]]: + """AIM validates against the OpenAI chat schema. Bare ``role: "tool"`` + without ``tool_call_id`` and bare ``role: "function"`` without ``name`` + are rejected; the flatten drops those fields, so any role outside + ``{system, user, assistant}`` collapses to ``user`` for the AIM POST.""" + safe_roles = {"system", "user", "assistant"} + return [{**m, "role": "user"} if m["role"] not in safe_roles else m for m in build_inspection_messages(data)] + @staticmethod def _rejection(message: str, *, openai_code: str | None = None) -> ProxyException: return ProxyException( @@ -177,7 +185,10 @@ class AimGuardrail(CustomGuardrail): user_email=user_email, litellm_call_id=call_id, ), - json={"messages": build_inspection_messages(request_data) + [{"role": "assistant", "content": output}]}, + json={ + "messages": self._build_aim_inspection_messages(request_data) + + [{"role": "assistant", "content": output}] + }, ) response.raise_for_status() res = response.json() diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e0eceaddd9..11f81f605ab 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -143,6 +143,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_web_search: Optional[bool] supports_reasoning: Optional[bool] supports_adaptive_thinking: Optional[bool] + supports_mid_conversation_system: Optional[bool] supports_url_context: Optional[bool] supports_none_reasoning_effort: Optional[bool] supports_minimal_reasoning_effort: Optional[bool] @@ -3368,6 +3369,7 @@ class LlmProviders(str, Enum): LIBERTAI = "libertai" PINSTRIPES = "pinstripes" DARKBLOOM = "darkbloom" + META = "meta" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" diff --git a/litellm/utils.py b/litellm/utils.py index 83d129339a4..73f247e3f3b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -61,7 +61,7 @@ from litellm._lazy_imports import ( ) from litellm._uuid import uuid from litellm.litellm_core_utils.fallback_generalizations import ( - match_fallback_generalization, + match_capability_generalizations, ) from litellm.constants import ( DEFAULT_CHAT_COMPLETION_PARAM_VALUES, @@ -2619,8 +2619,9 @@ _CACHE_PRICING_FIELDS = ( def _resolve_builtin_model_cost_entry(key: str, provider: str) -> Optional[Dict[str, Any]]: """Best-effort lookup of a built-in ``model_cost`` entry for a custom key - whose shape ``get_model_info`` cannot resolve (double provider prefixes - like ``bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region aliases). + whose shape ``get_model_info`` cannot resolve (repeated provider prefixes + like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region + aliases). Returns a copy of the matching entry so the caller can inherit its defaults (most importantly cache pricing) without mutating the shared built-in. @@ -2650,6 +2651,26 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> Optional[Dict[ return None +def _get_builtin_model_info_for_registration(model: str) -> Optional[ModelInfo]: + """Resolve ``model`` to its built-in cost-map entry for registration merging. + + Returns ``None`` when the lookup raises or when it resolved via a + fallback-generalization capability rule, detected as the resolved key missing + ``litellm.model_cost`` while matching a capability rule. A rule-derived entry + carries no pricing, so treating it as a hit would skip the built-in + cache-pricing inheritance for prefix-mangled keys. + """ + try: + info = get_model_info(model=model) + except Exception: + return None + if info["key"] in litellm.model_cost: + return info + if match_capability_generalizations(info["key"]) is None: + return info + return None + + def register_model(model_cost: Union[str, dict]): """ Register new / Override existing models (and their pricing) to specific providers. @@ -2690,10 +2711,11 @@ def register_model(model_cost: Union[str, dict]): existing_model = litellm.model_cost.get(key, {}) model_cost_key = key else: - try: - existing_model = cast(dict, get_model_info(model=key)) + builtin_model_info = _get_builtin_model_info_for_registration(model=_key_str) + if builtin_model_info is not None: + existing_model = cast(dict, builtin_model_info) model_cost_key = existing_model["key"] - except Exception: + else: existing_model = {} model_cost_key = key builtin_entry = _resolve_builtin_model_cost_entry(key=_key_str, provider=provider) @@ -5042,26 +5064,35 @@ def _get_model_info_from_generalization( potential_model_names: PotentialModelNamesAndCustomLLMProvider, custom_llm_provider: Optional[str], ) -> Optional[tuple[str, dict]]: - """Resolve an unmapped model via a declarative fallback-generalization rule. + """Resolve an unmapped model via the declarative capability generalization rules. Tries the same name candidates as the exact lookups, in the same order, and - returns ``(matched_name, model_info)`` for the first candidate whose rule also - satisfies the provider constraint. O(number of rules); only call after the + returns ``(matched_name, model_info)`` for the first candidate matched by at + least one capability rule, with ``litellm_provider`` backfilled from the + provider the caller requested. Rules lose to exact entries: if ANY candidate is + an exact ``litellm.model_cost`` key (necessarily provider-mismatched, or the + exact lookups would have returned it), the model is known rather than unmapped, + and resolving it from rules would hand an unpriced rule-derived entry to + callers whose fallback ladder (e.g. the cost calculator's model-name variants) + still had a priced exact name to try. O(number of rules); only call after the exact lookups have missed. """ - candidates = [ + candidates = ( potential_model_names["combined_model_name"], model, + potential_model_names["split_model"], potential_model_names["combined_stripped_model_name"], potential_model_names["stripped_model_name"], - potential_model_names["split_model"], - ] + ) + if any(_get_model_cost_key(candidate) is not None for candidate in candidates): + return None for candidate in candidates: - generalized_info = match_fallback_generalization(candidate) - if generalized_info is not None and _check_provider_match( - model_info=generalized_info, custom_llm_provider=custom_llm_provider - ): + generalized_info = match_capability_generalizations(candidate) + if generalized_info is None: + continue + if custom_llm_provider is None: return candidate, generalized_info + return candidate, {**generalized_info, "litellm_provider": custom_llm_provider} return None @@ -5094,6 +5125,11 @@ def _get_potential_model_names( stripped_model_name, ) + if custom_llm_provider in ("bedrock", "bedrock_converse"): + from litellm.llms.bedrock.common_utils import strip_bedrock_routing_prefix + + split_model = strip_bedrock_routing_prefix(split_model) + return PotentialModelNamesAndCustomLLMProvider( split_model=split_model, combined_model_name=combined_model_name, @@ -5261,9 +5297,9 @@ def _get_model_info_helper( Check if: (in order of specificity) 1. 'custom_llm_provider/model' in litellm.model_cost. Checks "groq/llama3-8b-8192" if model="llama3-8b-8192" and custom_llm_provider="groq" 2. 'model' in litellm.model_cost. Checks "gemini-1.5-pro-002" in litellm.model_cost if model="gemini-1.5-pro-002" and custom_llm_provider=None - 3. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given. - 4. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given. - 5. 'split_model' in litellm.model_cost. Checks "llama3-8b-8192" in litellm.model_cost if model="groq/llama3-8b-8192" + 3. 'split_model' in litellm.model_cost. Checks "au.anthropic.claude-opus-4-8" in litellm.model_cost if model="bedrock/au.anthropic.claude-opus-4-8" + 4. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given. + 5. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given. """ _model_info: Optional[Dict[str, Any]] = None @@ -5289,6 +5325,16 @@ def _get_model_info_helper( custom_llm_provider=model_cost_custom_llm_provider, ): _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(split_model) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, + custom_llm_provider=model_cost_custom_llm_provider, + ): + _model_info = None if _model_info is None: _matched_key = _get_model_cost_key(combined_stripped_model_name) if _matched_key is not None: @@ -5309,16 +5355,6 @@ def _get_model_info_helper( custom_llm_provider=model_cost_custom_llm_provider, ): _model_info = None - if _model_info is None: - _matched_key = _get_model_cost_key(split_model) - if _matched_key is not None: - key = _matched_key - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, - custom_llm_provider=model_cost_custom_llm_provider, - ): - _model_info = None if _model_info is None: generalization = _get_model_info_from_generalization( @@ -5466,6 +5502,7 @@ def _get_model_info_helper( supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), + supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None), supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None), supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), @@ -8022,6 +8059,16 @@ class ProviderConfigManager: ) return GithubCopilotAnthropicMessagesConfig() + + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + json_provider = JSONProviderRegistry.get(provider.value) + if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints: + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + return JSONProviderAnthropicMessagesConfig(json_provider) return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3dcdaed3f7..fb5746c2078 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1359,6 +1359,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1393,6 +1394,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1427,6 +1429,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1461,6 +1464,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1481,6 +1485,7 @@ "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1516,6 +1521,7 @@ "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1551,6 +1557,7 @@ "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1586,6 +1593,7 @@ "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1621,6 +1629,7 @@ "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1703,6 +1712,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1737,6 +1747,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1771,6 +1782,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1805,6 +1817,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1839,6 +1852,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1873,6 +1887,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -25075,6 +25090,42 @@ "supports_function_calling": true, "supports_tool_choice": false }, + "meta/muse-spark-1.1": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "meta", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.25e-06, + "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "meta_llama/Llama-3.3-70B-Instruct": { "litellm_provider": "meta_llama", "max_input_tokens": 128000, @@ -44041,20 +44092,26 @@ "fallback_generalizations": { "rules": [ { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", + "name": "bedrock-claude-ids", + "pattern": "anthropic\\.claude-", + "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.", "model_info": { - "supports_adaptive_thinking": true + "litellm_provider": "bedrock" } }, { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "name": "anthropic-claude-ids", + "pattern": "^claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?$", + "description": "A bare Claude family-major id with an optional minor and an optional 8-digit date suffix, anchored to the whole name, so claude-newfamily-5 routes like claude-newfamily-5-1 does. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.", + "model_info": { + "litellm_provider": "anthropic" + } + }, + { + "name": "claude-family-baseline", + "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", "model_info": { - "litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, @@ -44070,6 +44127,22 @@ "supports_pdf_input": true, "supports_system_messages": true } + }, + { + "name": "claude-adaptive-thinking", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "claude-mid-conversation-system", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "model_info": { + "supports_mid_conversation_system": true + } } ] } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 3034ada56ba..65db63dc045 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1984,6 +1984,23 @@ "interactions": true } }, + "meta": { + "display_name": "Meta Model API (`meta`)", + "url": "https://docs.litellm.ai/docs/providers/meta", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "pinstripes": { "display_name": "Pinstripes (`pinstripes`)", "url": "https://docs.litellm.ai/docs/providers/pinstripes", diff --git a/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py index 98ae7148c77..ef74249ca8e 100644 --- a/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py +++ b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py @@ -12,39 +12,39 @@ class TestMapReasoningEffort: def test_none_returns_none_for_opus_4_6(self): """reasoning_effort=None should return None for Opus 4.6, not adaptive.""" result = AnthropicConfig._map_reasoning_effort( - reasoning_effort=None, model="claude-opus-4-6" + reasoning_effort=None, model="claude-opus-4-6", custom_llm_provider="anthropic" ) assert result is None def test_none_returns_none_for_other_models(self): """reasoning_effort=None should return None for non-Opus models.""" result = AnthropicConfig._map_reasoning_effort( - reasoning_effort=None, model="claude-4-sonnet-20250514" + reasoning_effort=None, model="claude-4-sonnet-20250514", custom_llm_provider="anthropic" ) assert result is None def test_opus_4_6_returns_adaptive_for_low(self): result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="low", model="claude-opus-4-6" + reasoning_effort="low", model="claude-opus-4-6", custom_llm_provider="anthropic" ) assert result["type"] == "adaptive" def test_opus_4_6_returns_adaptive_for_high(self): result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="high", model="claude-opus-4-6" + reasoning_effort="high", model="claude-opus-4-6", custom_llm_provider="anthropic" ) assert result["type"] == "adaptive" def test_other_model_low_returns_enabled_with_budget(self): result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="low", model="claude-4-sonnet-20250514" + reasoning_effort="low", model="claude-4-sonnet-20250514", custom_llm_provider="anthropic" ) assert result["type"] == "enabled" assert "budget_tokens" in result def test_other_model_high_returns_enabled_with_budget(self): result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="high", model="claude-4-sonnet-20250514" + reasoning_effort="high", model="claude-4-sonnet-20250514", custom_llm_provider="anthropic" ) assert result["type"] == "enabled" assert "budget_tokens" in result @@ -52,13 +52,13 @@ class TestMapReasoningEffort: def test_none_string_returns_none_for_opus_4_6(self): """reasoning_effort='none' should return None for Opus 4.6.""" result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="none", model="claude-opus-4-6" + reasoning_effort="none", model="claude-opus-4-6", custom_llm_provider="anthropic" ) assert result is None def test_none_string_returns_none_for_other_models(self): """reasoning_effort='none' should return None for non-Opus models.""" result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="none", model="claude-4-sonnet-20250514" + reasoning_effort="none", model="claude-4-sonnet-20250514", custom_llm_provider="anthropic" ) assert result is None diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index 9ecb639fc3a..4c3e13da17a 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -509,10 +509,10 @@ def shipped_generalizations(): class TestClaudeModelPatternMatching: """ - The ``anthropic-claude`` fallback generalization rule routes future Claude - models to the Anthropic provider without requiring a + The ``anthropic-claude-ids`` fallback generalization routing rule routes future + Claude models to the Anthropic provider without requiring a model_prices_and_context_window.json entry. These tests exercise the rule - end-to-end through ``get_llm_provider`` and ``match_fallback_generalization``. + end-to-end through ``get_llm_provider`` and ``match_routing_generalization``. """ @pytest.mark.parametrize( @@ -556,10 +556,10 @@ class TestClaudeModelPatternMatching: self, model, shipped_generalizations ): from litellm.litellm_core_utils.fallback_generalizations import ( - match_fallback_generalization, + match_routing_generalization, ) - assert match_fallback_generalization(model) is None + assert match_routing_generalization(model) is None def test_routing_comes_from_the_rule_not_python(self, shipped_generalizations): """With the rule cleared, an unknown claude must no longer route to diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 298047ec18b..8f993aa385e 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -485,6 +485,74 @@ def test_build_span_exporter_variants(): OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") ) assert "OTLPSpanExporter" in type(http_exporter).__name__ + + +def test_otlp_logs_endpoint_normalization(): + norm = providers._otlp_logs_endpoint + # A base endpoint gets the signal path appended (the common OTLP env shape). + assert norm("http://collector:4318") == "http://collector:4318/v1/logs" + assert norm("http://collector:4318/") == "http://collector:4318/v1/logs" + # An already-correct path is left intact. + assert norm("http://collector:4318/v1/logs") == "http://collector:4318/v1/logs" + # A sibling signal's path is rewritten to logs, so one OTEL_ENDPOINT works + # for every signal rather than POSTing events at the traces path. + assert norm("http://collector:4318/v1/traces") == "http://collector:4318/v1/logs" + assert norm("http://collector:4318/v1/metrics") == "http://collector:4318/v1/logs" + assert norm(None) is None + + +def test_build_log_exporter_variants(): + from opentelemetry.sdk._logs.export import ConsoleLogExporter, InMemoryLogExporter + + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="console")), + ConsoleLogExporter, + ) + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="in_memory")), + InMemoryLogExporter, + ) + # An unrecognized kind falls back to console rather than dropping events. + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="unknown")), + ConsoleLogExporter, + ) + http_exporter = providers.build_log_exporter( + OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") + ) + assert "OTLPLogExporter" in type(http_exporter).__name__ + + +def test_build_logger_provider_picks_processor_by_exporter_kind(): + """Console and in-memory exporters export synchronously (tests depend on it); + every other destination gets the batch processor.""" + from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, + ConsoleLogExporter, + InMemoryLogExporter, + SimpleLogRecordProcessor, + ) + + cfg = OpenTelemetryV2Config(exporter="in_memory") + + def processor_of(provider): + return provider._multi_log_record_processor._log_record_processors[0] + + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=InMemoryLogExporter())), + SimpleLogRecordProcessor, + ) + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=ConsoleLogExporter())), + SimpleLogRecordProcessor, + ) + http_exporter = providers.build_log_exporter( + OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") + ) + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=http_exporter)), + BatchLogRecordProcessor, + ) grpc_exporter = providers.build_span_exporter( OpenTelemetryV2Config(exporter="otlp_grpc", endpoint="http://h:4317") ) @@ -721,6 +789,177 @@ def test_success_span_records_no_exception_event(): assert all(e.name != ExceptionEvent.NAME for e in span.events) +def _engine_with_event_recorder(): + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + from litellm.integrations.otel.emitter import SpanEmitter + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + provider, span_exporter = providers.in_memory_provider(cfg) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider)) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg, event_recorder=recorder) + return engine, span_exporter, log_exporter + + +def _llm_call_data(error): + return LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model=None, + response_id=None, + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=(), + error=error, + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + + +def test_operation_exception_log_event_emitted_on_failed_llm_call(): + """A failed LLM call records the GenAI semconv ``gen_ai.client.operation.exception`` + event on the logs signal: severity WARN, the full ``exception.*`` trio (including + the stacktrace, which span-side only exists under a vendor key), correlated to + the failed span via trace/span ids. The span-side error surface stays intact.""" + from opentelemetry._logs.severity import SeverityNumber + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.LLM_CALL, + _llm_call_data( + SpanError( + error_type="RateLimitError", + message="rate limited", + code="429", + stack_trace="Traceback (most recent call last) ...", + llm_provider="openai", + ) + ), + ) + (span,) = span_exporter.get_finished_spans() + (log,) = log_exporter.get_finished_logs() + record = log.log_record + + assert record.attributes["event.name"] == GenAIEvent.OPERATION_EXCEPTION + assert record.severity_number == SeverityNumber.WARN + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited" + assert record.attributes[ExceptionEvent.STACKTRACE] == "Traceback (most recent call last) ..." + assert record.trace_id == span.context.trace_id + assert record.span_id == span.context.span_id + + assert [e.name for e in span.events] == [ExceptionEvent.NAME] + assert span.attributes["error.type"] == "RateLimitError" + + +def test_operation_exception_log_event_omits_absent_stacktrace(): + from litellm.integrations.otel.model.semconv import ExceptionEvent + + engine, _, log_exporter = _engine_with_event_recorder() + engine.emit(SpanRole.LLM_CALL, _llm_call_data(SpanError(error_type="APIError", message="boom"))) + (log,) = log_exporter.get_finished_logs() + + assert ExceptionEvent.STACKTRACE not in log.log_record.attributes + assert log.log_record.attributes[ExceptionEvent.MESSAGE] == "boom" + + +def test_operation_exception_log_event_always_carries_required_pair(): + """``exception.type`` and ``exception.message`` are the semconv-required pair: + they ride the event even when the recorder is handed empty strings, so an + event is never emitted with no required field. Only the stacktrace is + conditional.""" + from opentelemetry.sdk._logs.export import InMemoryLogExporter + from opentelemetry.trace import INVALID_SPAN_CONTEXT + + from litellm.integrations.otel.model.semconv import ExceptionEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider)) + + recorder.record_operation_exception( + span_context=INVALID_SPAN_CONTEXT, + error_type="", + message="", + stack_trace="", + timestamp_ns=None, + ) + (log,) = log_exporter.get_finished_logs() + attributes = log.log_record.attributes + assert attributes[ExceptionEvent.TYPE] == "" + assert attributes[ExceptionEvent.MESSAGE] == "" + assert ExceptionEvent.STACKTRACE not in attributes + + +def test_operation_exception_log_event_not_emitted_on_success(): + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) + + assert len(span_exporter.get_finished_spans()) == 1 + assert log_exporter.get_finished_logs() == () + + +def test_operation_exception_log_event_only_for_llm_call_role(): + """The event is scoped to GenAI client operations; a failed guardrail span + keeps its span-side error surface but records no GenAI exception event.""" + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.GUARDRAIL, + GuardrailSpanData("presidio", status="failure", error=SpanError(error_type="X", message="denied")), + ) + (span,) = span_exporter.get_finished_spans() + + assert span.attributes["error.type"] == "X" + assert log_exporter.get_finished_logs() == () + + +def test_resolve_logger_provider_honors_explicit_noop_optout(monkeypatch): + """A ``NoOpLoggerProvider`` global is an explicit operator opt-out from the logs + signal: resolve to ``None`` so no recorder (and so no event) is ever built, + rather than emitting into a provider that drops everything.""" + from opentelemetry import _logs + from opentelemetry._logs import NoOpLoggerProvider + + from litellm.integrations.otel.logger import OpenTelemetryV2 + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + tracer_provider, _ = providers.in_memory_provider(cfg) + monkeypatch.setattr(_logs, "get_logger_provider", lambda: NoOpLoggerProvider()) + + assert providers.resolve_logger_provider(cfg) is None + logger = OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider) + assert logger._emitter._event_recorder is None + + +def test_resolve_logger_provider_reuses_operator_sdk_global(monkeypatch): + """Events ride an operator-configured logs pipeline rather than a second one + built by litellm, so they land wherever the operator's other logs land.""" + from opentelemetry import _logs + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + operator_provider = providers.build_logger_provider(cfg, log_exporter=InMemoryLogExporter()) + monkeypatch.setattr(_logs, "get_logger_provider", lambda: operator_provider) + + assert providers.resolve_logger_provider(cfg) is operator_provider + + +def test_operation_exception_event_keys_are_pinned(): + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + assert GenAIEvent.OPERATION_EXCEPTION == "gen_ai.client.operation.exception" + assert ExceptionEvent.STACKTRACE == "exception.stacktrace" + + # --- service taxonomy: which calls become spans, and of what kind ----------- # diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 697b9293eea..b5e077e3561 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -217,6 +217,61 @@ def test_async_log_failure_event_marks_error_status(): assert span.attributes["error.type"] == "RateLimitError" +def _logger_with_events(enable_events): + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=enable_events) + span_exporter = InMemorySpanExporter() + tracer_provider = providers.build_tracer_provider(cfg, exporter=span_exporter) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + logger = OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider, logger_provider=logger_provider) + return logger, span_exporter, log_exporter + + +def test_enable_events_records_operation_exception_through_failure_callback(): + """With ``enable_events`` on, a real failure callback records the GenAI + ``gen_ai.client.operation.exception`` log event, carrying the traceback from + the standard logging payload and correlated to the LLM-call span.""" + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + logger, span_exporter, log_exporter = _logger_with_events(enable_events=True) + payload = _payload( + status="failure", + error_information={ + "error_class": "RateLimitError", + "error_message": "429 rate limited", + "traceback": "Traceback (most recent call last) ...", + }, + ) + _emit_llm(logger, _kwargs(payload=payload), fail=True) + + (span,) = span_exporter.get_finished_spans() + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.attributes["event.name"] == GenAIEvent.OPERATION_EXCEPTION + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "429 rate limited" + assert record.attributes[ExceptionEvent.STACKTRACE] == "Traceback (most recent call last) ..." + assert record.trace_id == span.context.trace_id + assert record.span_id == span.context.span_id + + +def test_events_off_by_default_records_no_log_event_on_failure(): + """``enable_events`` defaults to off: even with a logs pipeline injected, a + failure records only the span-side error surface, no log event.""" + logger, span_exporter, log_exporter = _logger_with_events(enable_events=False) + payload = _payload( + status="failure", + error_information={"error_class": "RateLimitError", "error_message": "429"}, + ) + _emit_llm(logger, _kwargs(payload=payload), fail=True) + + assert len(span_exporter.get_finished_spans()) == 1 + assert log_exporter.get_finished_logs() == () + assert OpenTelemetryV2Config(exporter="in_memory").enable_events is False + + def test_sync_log_event_is_noop(): """V2 closes the span async-only; the sync callback runs out-of-context, so it no-ops (the span stays open on the carrier until the async callback).""" diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 410014958b6..0414836fa79 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -1,11 +1,13 @@ """ Tests for the declarative fallback-generalizations mechanism. -Covers both the pure module (litellm.litellm_core_utils.fallback_generalizations) -and its end-to-end wiring into provider routing (get_llm_provider) and model-info -resolution (get_model_info / supports_*). +Covers the pure module (litellm.litellm_core_utils.fallback_generalizations): the +routing/capability rule split, install-time validation, capability unioning; and +its end-to-end wiring into provider routing (get_llm_provider) and model-info +resolution (get_model_info) including the shipped rules in the bundled cost map. """ +import logging import os import sys @@ -14,9 +16,11 @@ import pytest sys.path.insert(0, os.path.abspath("../../..")) import litellm +from litellm._logging import verbose_logger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, - match_fallback_generalization, + match_capability_generalizations, + match_routing_generalization, set_fallback_generalizations, ) @@ -31,50 +35,205 @@ def restore_generalizations(): set_fallback_generalizations(previous) +class _RecordingHandler(logging.Handler): + def __init__(self): + super().__init__(level=logging.WARNING) + self.messages = [] + + def emit(self, record): + self.messages.append(record.getMessage()) + + +@pytest.fixture +def warning_messages(): + handler = _RecordingHandler() + previous_level = verbose_logger.level + verbose_logger.setLevel(logging.WARNING) + verbose_logger.addHandler(handler) + try: + yield handler.messages + finally: + verbose_logger.removeHandler(handler) + verbose_logger.setLevel(previous_level) + + # --------------------------------------------------------------------------- # -# Pure module behaviour +# Engine: routing rules # --------------------------------------------------------------------------- # -def test_match_returns_model_info_of_first_matching_rule(restore_generalizations): +def test_routing_inference_first_match_wins(restore_generalizations): + restore_generalizations( + [ + {"name": "first", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}}, + {"name": "second", "pattern": r"^acme-pro-", "model_info": {"litellm_provider": "anthropic"}}, + ] + ) + assert match_routing_generalization("acme-pro-1") == "openai" + assert match_routing_generalization("gpt-4o") is None + assert match_routing_generalization("") is None + + +def test_capability_rules_do_not_route(restore_generalizations): + restore_generalizations([{"name": "caps", "pattern": r"^acme-", "model_info": {"supports_vision": True}}]) + assert match_routing_generalization("acme-pro-1") is None + + +def test_routing_match_is_case_insensitive(restore_generalizations): + restore_generalizations( + [{"name": "r", "pattern": r"^claude-opus", "model_info": {"litellm_provider": "anthropic"}}] + ) + assert match_routing_generalization("CLAUDE-OPUS-9-9") == "anthropic" + + +# --------------------------------------------------------------------------- # +# Engine: capability rules +# --------------------------------------------------------------------------- # + + +def test_capability_union_is_last_wins_in_file_order(restore_generalizations): restore_generalizations( [ { - "name": "first", + "name": "broad", "pattern": r"^acme-", - "model_info": {"litellm_provider": "openai", "tag": "first"}, + "model_info": {"mode": "chat", "supports_vision": True, "max_input_tokens": 1000}, }, { - "name": "second", + "name": "narrow", "pattern": r"^acme-pro-", - "model_info": {"litellm_provider": "anthropic", "tag": "second"}, + "model_info": {"supports_vision": False, "supports_reasoning": True}, }, ] ) - # Both rules match "acme-pro-1"; first-in-list wins (documented precedence). - matched = match_fallback_generalization("acme-pro-1") - assert matched is not None - assert matched["tag"] == "first" + assert match_capability_generalizations("acme-pro-1") == { + "mode": "chat", + "supports_vision": False, + "max_input_tokens": 1000, + "supports_reasoning": True, + } + assert match_capability_generalizations("acme-basic-1") == { + "mode": "chat", + "supports_vision": True, + "max_input_tokens": 1000, + } -def test_match_is_case_insensitive(restore_generalizations): +def test_routing_rules_are_excluded_from_capability_results(restore_generalizations): restore_generalizations( - [{"name": "r", "pattern": r"^claude-opus", "model_info": {"ok": True}}] + [ + {"name": "route", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}}, + {"name": "caps", "pattern": r"^acme-pro-", "model_info": {"supports_vision": True}}, + ] ) - assert match_fallback_generalization("CLAUDE-OPUS-9-9") == {"ok": True} + assert match_capability_generalizations("acme-pro-1") == {"supports_vision": True} + assert match_capability_generalizations("acme-basic-1") is None -def test_no_match_returns_none(restore_generalizations): - restore_generalizations( - [{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}] - ) - assert match_fallback_generalization("gpt-4o") is None - assert match_fallback_generalization("") is None - - -def test_empty_rules_match_nothing(restore_generalizations): +def test_no_capability_match_returns_none(restore_generalizations): + restore_generalizations([{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}]) + assert match_capability_generalizations("gpt-4o") is None + assert match_capability_generalizations("") is None restore_generalizations([]) - assert match_fallback_generalization("claude-opus-9-9") is None + assert match_capability_generalizations("claude-opus-9-9") is None + + +def test_reinstalling_rules_replaces_compiled_rules(restore_generalizations): + restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}]) + assert match_capability_generalizations("aaa-1") == {"v": 1} + set_fallback_generalizations([{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}]) + assert match_capability_generalizations("aaa-1") is None + assert match_capability_generalizations("bbb-1") == {"v": 2} + + +# --------------------------------------------------------------------------- # +# Engine: install-time validation and legacy-schema shim +# --------------------------------------------------------------------------- # + + +def test_legacy_mixed_rule_acts_as_both_kinds(restore_generalizations): + """A legacy rule mixing ``litellm_provider`` with capability keys routes AND + contributes its full model_info (provider included) to the capability union.""" + restore_generalizations( + [ + { + "name": "legacy-mixed", + "pattern": r"^acme-", + "model_info": {"litellm_provider": "anthropic", "supports_vision": True}, + }, + {"name": "new-caps", "pattern": r"^acme-pro-", "model_info": {"supports_reasoning": True}}, + ] + ) + assert match_routing_generalization("acme-pro-1") == "anthropic" + assert match_capability_generalizations("acme-pro-1") == { + "litellm_provider": "anthropic", + "supports_vision": True, + "supports_reasoning": True, + } + + +LEGACY_MAIN_RULES = [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": {"supports_adaptive_thinking": True}, + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": True, + "supports_parallel_function_calling": True, + "supports_vision": True, + "supports_tool_choice": True, + "supports_assistant_prefill": True, + "supports_prompt_caching": True, + "supports_response_schema": True, + "supports_reasoning": True, + "supports_pdf_input": True, + "supports_system_messages": True, + }, + }, +] + + +def test_legacy_main_schema_keeps_unmapped_claude_working(restore_generalizations): + """Pins the remote-map transition window: a released proxy running this engine + against main's old-schema block (mixed provider+capability rule plus ``extends``, + copied verbatim above) must keep unmapped-Claude inference and info resolution + working until the new-schema JSON reaches main.""" + restore_generalizations([dict(rule) for rule in LEGACY_MAIN_RULES]) + litellm.get_model_info.cache_clear() + + _, provider, _, _ = litellm.get_llm_provider(model="claude-opus-9-9") + assert provider == "anthropic" + + info = litellm.get_model_info("claude-opus-9-9") + assert info["litellm_provider"] == "anthropic" + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + assert info["max_input_tokens"] == 200000 + assert not info.get("input_cost_per_token") + + low = litellm.get_model_info("claude-opus-4-0") + assert low["litellm_provider"] == "anthropic" + assert low["supports_function_calling"] is True + assert low.get("supports_adaptive_thinking") is None + + +def test_non_string_provider_rule_warns_and_is_skipped(restore_generalizations, warning_messages): + restore_generalizations([{"name": "bad-provider", "pattern": r"^acme-", "model_info": {"litellm_provider": 42}}]) + assert any("bad-provider" in message for message in warning_messages) + assert match_routing_generalization("acme-1") is None + assert match_capability_generalizations("acme-1") is None def test_malformed_rules_are_skipped_not_fatal(restore_generalizations): @@ -88,69 +247,7 @@ def test_malformed_rules_are_skipped_not_fatal(restore_generalizations): {"name": "good", "pattern": r"^claude-", "model_info": {"good": True}}, ] ) - # Non-dict entries and dicts with bad fields are all skipped; the one - # valid rule still matches. - assert match_fallback_generalization("claude-opus-9-9") == {"good": True} - - -def test_setting_rules_invalidates_compiled_cache(restore_generalizations): - restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}]) - assert match_fallback_generalization("aaa-1") == {"v": 1} - # Re-install different rules; the compiled cache must be rebuilt. - set_fallback_generalizations( - [{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}] - ) - assert match_fallback_generalization("aaa-1") is None - assert match_fallback_generalization("bbb-1") == {"v": 2} - - -def test_extends_inherits_parent_and_own_overrides(restore_generalizations): - """A rule's ``extends`` pulls in the parent's model_info; its own keys win on conflict, - so a narrow rule carries only its delta instead of duplicating the parent.""" - restore_generalizations( - [ - { - "name": "base", - "pattern": r"^base-only$", - "model_info": { - "litellm_provider": "anthropic", - "input_cost_per_token": 5e-06, - "supports_vision": True, - }, - }, - { - "name": "child", - "pattern": r"^kid-", - "extends": "base", - "model_info": { - "supports_adaptive_thinking": True, - "supports_vision": False, - }, - }, - ] - ) - matched = match_fallback_generalization("kid-1") - assert matched == { - "litellm_provider": "anthropic", - "input_cost_per_token": 5e-06, - "supports_vision": False, - "supports_adaptive_thinking": True, - } - - -def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalizations): - """A dangling ``extends`` is non-fatal: the rule resolves to its own model_info.""" - restore_generalizations( - [ - { - "name": "orphan", - "pattern": r"^orphan-", - "extends": "does-not-exist", - "model_info": {"litellm_provider": "openai"}, - } - ] - ) - assert match_fallback_generalization("orphan-1") == {"litellm_provider": "openai"} + assert match_capability_generalizations("claude-opus-9-9") == {"good": True} # --------------------------------------------------------------------------- # @@ -158,91 +255,61 @@ def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalization # --------------------------------------------------------------------------- # -@pytest.fixture -def myco_rule(restore_generalizations): - """A self-contained rule carrying provider, pricing, context and capabilities.""" +def test_unknown_model_routes_via_routing_rule(restore_generalizations): + restore_generalizations([{"name": "myco", "pattern": r"^myco-", "model_info": {"litellm_provider": "openai"}}]) + _, provider, _, _ = litellm.get_llm_provider(model="myco-fast-1") + assert provider == "openai" + + +def test_capability_info_backfills_requested_provider(restore_generalizations): restore_generalizations( [ { - "name": "myco", - "pattern": r"^myco-[a-z]+-\d+$", + "name": "beeco-caps", + "pattern": r"^beeco-[a-z]+-\d+$", "model_info": { - "litellm_provider": "openai", "mode": "chat", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, "max_input_tokens": 12345, - "max_output_tokens": 678, "supports_vision": True, "supports_function_calling": True, }, } ] ) - return "myco-fast-1" - - -def test_unknown_model_routes_via_rule(myco_rule): - _, provider, _, _ = litellm.get_llm_provider(model=myco_rule) - assert provider == "openai" - - -def test_unknown_model_gets_pricing_context_and_capabilities(myco_rule): - info = litellm.get_model_info(myco_rule) - assert info["litellm_provider"] == "openai" - assert info["input_cost_per_token"] == 1e-06 - assert info["output_cost_per_token"] == 2e-06 + litellm.get_model_info.cache_clear() + info = litellm.get_model_info("beeco-fast-1", custom_llm_provider="groq") + assert info["litellm_provider"] == "groq" assert info["max_input_tokens"] == 12345 assert info["supports_vision"] is True + other = litellm.get_model_info("beeco-fast-1", custom_llm_provider="openai") + assert other["litellm_provider"] == "openai" -def test_supports_helper_reads_through_generalization(myco_rule): - assert litellm.supports_vision(myco_rule) is True - assert litellm.supports_function_calling(myco_rule) is True +def test_routing_only_match_does_not_resolve_model_info(restore_generalizations): + restore_generalizations([{"name": "route", "pattern": r"^ceeco-", "model_info": {"litellm_provider": "openai"}}]) + litellm.get_model_info.cache_clear() + with pytest.raises(Exception): + litellm.get_model_info("ceeco-fast-1", custom_llm_provider="openai") def test_exact_entry_takes_precedence_over_rule(restore_generalizations): - """An exact cost-map entry must win over a rule that also matches it.""" restore_generalizations( - [ - { - "name": "shadow-gpt4o", - "pattern": r"^gpt-4o$", - "model_info": { - "litellm_provider": "anthropic", - "input_cost_per_token": 999.0, - }, - } - ] + [{"name": "shadow-gpt4o", "pattern": r"^gpt-4o$", "model_info": {"input_cost_per_token": 999.0}}] ) + litellm.get_model_info.cache_clear() info = litellm.get_model_info("gpt-4o") - # Resolved from the real exact entry, not the shadowing rule. assert info["litellm_provider"] == "openai" assert info["input_cost_per_token"] != 999.0 -def test_unknown_model_without_matching_rule_still_unmapped(restore_generalizations): - restore_generalizations( - [ - { - "name": "claude", - "pattern": r"^claude-", - "model_info": {"litellm_provider": "anthropic"}, - } - ] - ) - with pytest.raises(Exception): - litellm.get_model_info("totally-unknown-model-xyz") - - # --------------------------------------------------------------------------- # -# Shipped anthropic-claude rule +# Shipped rules (bundled cost map) # --------------------------------------------------------------------------- # @pytest.fixture def shipped_cost_map(monkeypatch): - """Activate the bundled cost map so the shipped anthropic-claude rule is installed.""" + """Activate the bundled cost map so the shipped rules are installed.""" original_cost = litellm.model_cost previous_rules = list(get_fallback_generalization_rules()) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -256,45 +323,241 @@ def shipped_cost_map(monkeypatch): set_fallback_generalizations(previous_rules) -def test_shipped_rule_marks_unmapped_high_version_claude_adaptive_without_pricing( - shipped_cost_map, -): - """An unmapped Claude >= 4.6 resolves via the version-gated adaptive-thinking rule, which - inherits routing and capabilities from the base rule and adds ``supports_adaptive_thinking``. - The rule carries no pricing, so cost stays unpriced (zero, not a fabricated number) rather - than reporting a confidently-wrong price.""" - model = "claude-opus-9-9" +def test_shipped_bare_claude_id_routes_to_anthropic(shipped_cost_map): + _, provider, _, _ = litellm.get_llm_provider(model="claude-haiku-4-6") + assert provider == "anthropic" + + +def test_shipped_bedrock_syntax_claude_id_routes_to_bedrock(shipped_cost_map): + """Regression: a bedrock-syntax id must infer bedrock even when its version also + matches an unanchored Anthropic capability pattern. The old first-match-wins engine + routed global.anthropic.claude-haiku-4-6 to anthropic via the adaptive rule.""" + for model in [ + "global.anthropic.claude-haiku-4-6", + "us.anthropic.claude-haiku-4-6", + "anthropic.claude-haiku-4-6", + "eu.anthropic.claude-opus-5-0", + ]: + assert model not in litellm.model_cost + _, provider, _, _ = litellm.get_llm_provider(model=model) + assert provider == "bedrock", model + + +def test_shipped_rules_resolve_unmapped_bedrock_claude_with_bedrock_provider(shipped_cost_map): + model = "us.anthropic.claude-haiku-4-6" assert model not in litellm.model_cost - info = litellm.get_model_info(model) - assert info["litellm_provider"] == "anthropic" + info = litellm.get_model_info(model, custom_llm_provider="bedrock") + assert info["litellm_provider"] == "bedrock" assert info["supports_adaptive_thinking"] is True assert info["supports_function_calling"] is True + assert info["max_input_tokens"] == 200000 + assert info.get("supports_mid_conversation_system") is None assert not info.get("input_cost_per_token") assert not info.get("output_cost_per_token") -def test_shipped_rule_resolves_unmapped_low_version_claude_without_adaptive(shipped_cost_map): - """An unmapped Claude < 4.6 falls through to the version-neutral anthropic-claude rule: it - gets provider routing and baseline capabilities but no ``supports_adaptive_thinking`` flag, - so a sub-4.6 alias such as ``claude-opus-4-0`` resolves yet is never marked adaptive.""" - model = "claude-opus-4-0" +def test_shipped_rules_stack_adaptive_and_mid_conversation_flags(shipped_cost_map): + model = "claude-opus-4-9" + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider="anthropic") + assert info["litellm_provider"] == "anthropic" + assert info["supports_adaptive_thinking"] is True + assert info["supports_mid_conversation_system"] is True + assert info["supports_function_calling"] is True + + +@pytest.mark.parametrize( + "model,provider", + [ + ("claude-opus-4-9@20260101", "vertex_ai"), + ("databricks-claude-opus-5-1", "databricks"), + ], +) +def test_shipped_rules_are_provider_neutral_for_unmapped_ids(shipped_cost_map, model, provider): + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info["litellm_provider"] == provider + assert info["supports_adaptive_thinking"] is True + assert info["supports_mid_conversation_system"] is True + assert info["supports_function_calling"] is True + + +@pytest.mark.parametrize( + "model,provider,adaptive,mid_conversation", + [ + ("us.anthropic.claude-opus-4-5", "bedrock", None, None), + ("claude-haiku-4-6", "anthropic", True, None), + ("claude-haiku-4-7", "anthropic", True, None), + ("claude-haiku-4-8", "anthropic", True, True), + ("claude-haiku-4-9", "anthropic", True, True), + ("claude-haiku-4-10", "anthropic", True, True), + ("claude-haiku-5-0", "anthropic", True, True), + ("claude-sonnet-5-1", "anthropic", True, True), + ], +) +def test_shipped_version_boundaries(shipped_cost_map, model, provider, adaptive, mid_conversation): + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info["litellm_provider"] == provider + assert info["supports_function_calling"] is True + assert not info.get("input_cost_per_token") + assert info.get("supports_adaptive_thinking") is adaptive, model + assert info.get("supports_mid_conversation_system") is mid_conversation, model + + +def test_shipped_rules_cover_new_families_like_fable_at_5_plus(shipped_cost_map): + """Both version gates accept any claude-- id at major 5 or higher, bare + major or major-minor, so a new family shaped like claude-fable-5 gets adaptive + thinking and mid-conversation system support without a cost-map entry.""" + model = "claude-fable-5-1" + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider="anthropic") + assert info["supports_mid_conversation_system"] is True + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + + +def test_shipped_rules_flag_bare_5_plus_majors_of_any_family(shipped_cost_map): + """A bare 5+ major with no minor gets both flags at the rule level; the mapped + claude-fable-5 entry itself still resolves from the cost map, so this pins the + pattern via the capability union rather than get_model_info.""" + matched = match_capability_generalizations("claude-fable-5") + assert matched is not None + assert matched["supports_adaptive_thinking"] is True + assert matched["supports_mid_conversation_system"] is True + + +def test_shipped_version_gates_are_family_agnostic_at_4x(shipped_cost_map): + """Both version gates apply to any claude-- id, 4.x included: a non-core + family at 4.9 gets adaptive and mid-conversation, while the same family at 4.5 + gets baseline only. Only opus/sonnet/haiku ever shipped 4.x ids, so the + family-agnostic 4.6+ gate changes nothing for real models.""" + high = litellm.get_model_info("claude-newfam-4-9", custom_llm_provider="anthropic") + assert high["supports_adaptive_thinking"] is True + assert high["supports_mid_conversation_system"] is True + assert high["supports_function_calling"] is True + + low = litellm.get_model_info("claude-newfam-4-5", custom_llm_provider="anthropic") + assert low.get("supports_adaptive_thinking") is None + assert low.get("supports_mid_conversation_system") is None + assert low["supports_function_calling"] is True + + +def test_shipped_rules_give_bare_majors_the_full_baseline_union(shipped_cost_map): + """A bare-major unmapped id (no minor) resolves the same baseline union as its + major-minor sibling: the baseline pattern's minor is optional, so claude-newt-5 + is not left with version flags but no mode, token limits, or capability facts.""" + model = "anthropic/claude-newt-5" assert model not in litellm.model_cost info = litellm.get_model_info(model) assert info["litellm_provider"] == "anthropic" + assert info["mode"] == "chat" + assert info["max_tokens"] == 64000 assert info["supports_function_calling"] is True - assert info.get("supports_adaptive_thinking") is None - assert not info.get("input_cost_per_token") + assert info["supports_adaptive_thinking"] is True + assert info["supports_mid_conversation_system"] is True + + +def test_shipped_routing_rule_covers_bare_majors(shipped_cost_map): + _, provider, _, _ = litellm.get_llm_provider(model="claude-newt-5") + assert provider == "anthropic" + + +def test_shipped_adaptive_rule_requires_claude_prefix(shipped_cost_map): + """A non-Claude name embedding a core-family 4.6+/5.x version substring must not + resolve from the rules; serving it a zero-priced rule entry would silently + swallow cost tracking for arbitrary custom deployment names.""" + model = "openai/team-sonnet-5-1-alias" + assert model not in litellm.model_cost + assert match_capability_generalizations("team-sonnet-5-1-alias") is None + with pytest.raises(Exception): + litellm.get_model_info(model) + + +def test_shipped_exact_entry_beats_rules(shipped_cost_map): + model = "us.anthropic.claude-sonnet-4-6" + assert model in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider="bedrock") + assert info["litellm_provider"] == "bedrock_converse" + assert info["input_cost_per_token"] == 3.3e-06 + assert info["max_input_tokens"] == 1000000 + assert info["supports_adaptive_thinking"] is True + assert info.get("supports_mid_conversation_system") is None + + +def test_shipped_rules_lose_to_exact_entries_across_cost_ladder_variants(shipped_cost_map): + """A route-mangled variant of an exactly-mapped model must never resolve from + rules. The cost calculator tries model-name variants in order; a rule-derived + unpriced entry served for an early variant (here bedrock/claude-haiku-4-5-20251001, + whose bare form is exactly mapped under anthropic) would zero out the bill even + though the exact priced bedrock entry is one variant later. An exactly-mapped id + under a mismatched provider raises instead of resolving from rules.""" + from litellm import completion_cost + from litellm.types.utils import ModelResponse, Usage + + assert "claude-haiku-4-5-20251001" in litellm.model_cost + with pytest.raises(Exception): + litellm.get_model_info("claude-haiku-4-5-20251001", custom_llm_provider="bedrock") + + entry = litellm.model_cost["us.anthropic.claude-haiku-4-5-20251001-v1:0"] + response = ModelResponse(model="claude-haiku-4-5-20251001", usage=Usage(prompt_tokens=100, completion_tokens=50)) + cost = completion_cost( + completion_response=response, + model="bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", + custom_llm_provider="bedrock", + ) + assert cost == 100 * entry["input_cost_per_token"] + 50 * entry["output_cost_per_token"] + assert cost > 0 def test_shipped_adaptive_rule_gates_on_version_not_pricing(shipped_cost_map): - """The version-gated ``anthropic-claude-adaptive-thinking`` rule marks an unmapped - Claude adaptive only from >= 4.6, including provider-prefixed ids the anchored pricing - rule cannot match, while leaving < 4.6 (and the dated Opus 4.0 form) non-adaptive.""" + """The version-gated adaptive-thinking capability rule marks an unmapped Claude + adaptive only from >= 4.6, including provider-prefixed ids the anchored routing + rule cannot match, while leaving the dated Opus 4.0 form non-adaptive.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo adaptive = "us.anthropic.claude-opus-4-9" non_adaptive = "us.anthropic.claude-opus-4-20250514" assert adaptive not in litellm.model_cost assert non_adaptive not in litellm.model_cost - assert AnthropicModelInfo._is_adaptive_thinking_model(adaptive) is True - assert AnthropicModelInfo._is_adaptive_thinking_model(non_adaptive) is False + assert AnthropicModelInfo._is_adaptive_thinking_model(adaptive, "anthropic") is True + assert AnthropicModelInfo._is_adaptive_thinking_model(non_adaptive, "anthropic") is False + + +def test_shipped_rules_resolve_unmapped_future_bedrock_claude_with_both_flags(shipped_cost_map): + """An unmapped Bedrock Claude >= 4.8 resolves for custom_llm_provider="bedrock" with + baseline capabilities, both version-gated flags, the bedrock provider backfilled, and + no fabricated pricing.""" + model = "us.anthropic.claude-opus-4-9" + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider="bedrock") + assert info["litellm_provider"] == "bedrock" + assert info["supports_mid_conversation_system"] is True + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + assert not info.get("input_cost_per_token") + + +def test_shipped_mid_conversation_gate_on_bedrock_ids(shipped_cost_map): + """Bedrock-syntax ids gain ``supports_mid_conversation_system`` only from 4.8 upward, + bare 5+ majors and new families included; 4.7-and-below Bedrock ids never gain it. + The flag comes from the provider-neutral capability rule rather than a bedrock-scoped + one, so the same gate covers native and vertex-shaped ids too.""" + for flagged in ( + "us.anthropic.claude-opus-4-8", + "jp.anthropic.claude-opus-4-8", + "anthropic.claude-sonnet-5", + "us.anthropic.claude-fable-5", + "anthropic.claude-sonnet-5-20260101-v1:0", + ): + matched = match_capability_generalizations(flagged) + assert matched is not None, flagged + assert matched["supports_mid_conversation_system"] is True, flagged + for unflagged in ( + "us.anthropic.claude-opus-4-7", + "us.anthropic.claude-sonnet-4-6", + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0", + ): + matched = match_capability_generalizations(unflagged) + assert matched is None or not matched.get("supports_mid_conversation_system"), unflagged diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 924aaa7775d..bdd71f28b1a 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -13,7 +13,8 @@ sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, - match_fallback_generalization, + match_capability_generalizations, + match_routing_generalization, set_fallback_generalizations, ) from litellm.litellm_core_utils.get_model_cost_map import ( @@ -102,9 +103,7 @@ def test_finalize_pops_key_and_installs_rules(): # The reserved key is removed from the returned model map ... assert FALLBACK_GENERALIZATIONS_KEY not in finalized # ... and its rules are installed into the generalizations module. - assert match_fallback_generalization("widget-9") == { - "litellm_provider": "openai" - } + assert match_routing_generalization("widget-9") == "openai" finally: set_fallback_generalizations(previous) @@ -116,27 +115,25 @@ def test_finalize_with_no_block_clears_rules(): [{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}] ) _finalize_model_cost_map(_make_models(2)) - assert match_fallback_generalization("x-1") is None + assert match_capability_generalizations("x-1") is None finally: set_fallback_generalizations(previous) -def test_shipped_backup_carries_the_anthropic_claude_rule(): - """The bundled backup must ship the anthropic-claude rule so a fresh install - (or an offline fallback) routes unknown Claude models without code changes.""" +def test_shipped_backup_carries_the_claude_routing_rules(): + """The bundled backup must ship the Claude routing rules so a fresh install + (or an offline fallback) routes unknown Claude models without code changes. + Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule.""" backup = GetModelCostMap.load_local_model_cost_map() rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", []) - names = {r.get("name") for r in rules} - assert "anthropic-claude" in names - - rule = next(r for r in rules if r.get("name") == "anthropic-claude") - assert rule["model_info"]["litellm_provider"] == "anthropic" + names = [r.get("name") for r in rules] + assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids") previous = list(get_fallback_generalization_rules()) try: set_fallback_generalizations(rules) - matched = match_fallback_generalization("claude-opus-4-9") - assert matched is not None and matched["litellm_provider"] == "anthropic" + assert match_routing_generalization("claude-opus-4-9") == "anthropic" + assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock" finally: set_fallback_generalizations(previous) @@ -147,23 +144,19 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): route) and on the version-gated anthropic-claude-adaptive-thinking rule for unmapped future Claudes, while leaving the dated Claude 4.0 names ("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor - version. The version-neutral anthropic-claude pricing rule must not flag it, so - an unmapped sub-4.6 name is priced but stays non-adaptive. The adaptive rule must - inherit pricing from the pricing rule via ``extends`` and carry only its delta, so - the Opus-tier price block is never duplicated across rules.""" + version. The version-neutral claude-family-baseline capability rule must not flag + it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule + carries only its delta; capability unioning stacks it onto the baseline, so the + baseline block is never duplicated across rules and no rule needs ``extends``.""" backup = GetModelCostMap.load_local_model_cost_map() rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"] - pricing_rule = next(r for r in rules if r.get("name") == "anthropic-claude") - adaptive_rule = next( - r for r in rules if r.get("name") == "anthropic-claude-adaptive-thinking" - ) - assert "supports_adaptive_thinking" not in pricing_rule["model_info"] - assert adaptive_rule["model_info"]["supports_adaptive_thinking"] is True - - assert "extends" not in pricing_rule - assert adaptive_rule.get("extends") == "anthropic-claude" + baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline") + adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking") + assert "supports_adaptive_thinking" not in baseline_rule["model_info"] + assert "litellm_provider" not in baseline_rule["model_info"] assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True} + assert all("extends" not in r for r in rules) for adaptive in [ "anthropic.claude-opus-4-8", diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 852479e81e4..7fb38544c52 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1661,7 +1661,7 @@ def test_effort_beta_header_injection(): # Test with effort parameter optional_params = {"output_config": {"effort": "low"}} - effort_used = model_info.is_effort_used(optional_params=optional_params) + effort_used = model_info.is_effort_used(optional_params=optional_params, custom_llm_provider="anthropic") assert effort_used is True headers = model_info.get_anthropic_headers( @@ -1877,7 +1877,7 @@ def test_anthropic_drop_params_false_forwards_to_unsupported_model(): ], ) def test_anthropic_model_supports_effort_param_recognizes_supporting_models(model): - assert AnthropicConfig._model_supports_effort_param(model) is True + assert AnthropicConfig._model_supports_effort_param(model, "anthropic") is True @pytest.mark.parametrize( @@ -1890,7 +1890,7 @@ def test_anthropic_model_supports_effort_param_recognizes_supporting_models(mode ], ) def test_anthropic_model_supports_effort_param_rejects_non_supporting_models(model): - assert AnthropicConfig._model_supports_effort_param(model) is False + assert AnthropicConfig._model_supports_effort_param(model, "anthropic") is False @pytest.mark.parametrize( @@ -2217,7 +2217,7 @@ def test_get_config_does_not_leak_module_constants(): ) def test_supports_effort_level_handles_provider_prefixes(model, level, expected): """``_supports_effort_level`` resolves bedrock/vertex/azure-prefixed model ids.""" - assert AnthropicConfig._supports_effort_level(model, level) is expected + assert AnthropicConfig._supports_effort_level(model, level, "anthropic") is expected @pytest.mark.parametrize( @@ -2239,7 +2239,7 @@ def test_supports_effort_level_handles_provider_prefixes(model, level, expected) def test_validate_effort_for_model_centralises_per_model_gating( model, effort, expect_error ): - err = AnthropicConfig._validate_effort_for_model(model, effort) + err = AnthropicConfig._validate_effort_for_model(model, effort, "anthropic") if expect_error: assert err is not None assert effort in err @@ -2490,7 +2490,7 @@ def test_is_adaptive_thinking_model_is_sourced_from_cost_map( fallback for ids the cost map cannot resolve. The dated Claude 4.0 names stay non-adaptive because the date suffix is not read as a minor version, while 4.8/4.9/5.x are covered without a code change.""" - assert AnthropicConfig._is_adaptive_thinking_model(model) is expected + assert AnthropicConfig._is_adaptive_thinking_model(model, "anthropic") is expected def test_get_supported_params_includes_reasoning_for_sonnet_4_6_alias( @@ -2836,6 +2836,7 @@ def test_effort_beta_header_not_injected_for_46_models(): result = model_info.is_effort_used( optional_params={"output_config": {"effort": "high"}}, model=model, + custom_llm_provider="anthropic", ) assert result is False, f"is_effort_used should return False for {model}" @@ -2947,6 +2948,7 @@ def test_effort_beta_header_still_injected_for_older_models(): result = model_info.is_effort_used( optional_params={"output_config": {"effort": "low"}}, model="claude-opus-4-5-20251101", + custom_llm_provider="anthropic", ) assert result is True diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_effort.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_effort.py new file mode 100644 index 00000000000..06d3effcfbb --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_effort.py @@ -0,0 +1,189 @@ +import pytest + +from litellm.constants import ( + DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, +) +from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) + + +def _claude_code_payload(effort="medium", max_tokens=8192, **output_config_extra): + """The exact adaptive-thinking shape Claude Code (claude-cli) sends.""" + output_config = {"effort": effort, **output_config_extra} + return { + "max_tokens": max_tokens, + "thinking": {"type": "adaptive"}, + "output_config": output_config, + } + + +def _transform(model, params, litellm_params=None): + return AnthropicMessagesConfig().transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=dict(params), + litellm_params=litellm_params or {}, + headers={}, + ) + + +def test_effort_translated_to_legacy_thinking_for_haiku_4_5(): + """Core regression: Claude Code sends adaptive thinking + effort to Haiku 4.5 + (thinking-capable, pre-4.6). Effort must be translated to legacy extended + thinking rather than forwarded raw (which Anthropic rejects with "This model + does not support the effort parameter").""" + result = _transform("claude-haiku-4-5", _claude_code_payload(effort="medium")) + + assert result["thinking"] == { + "type": "enabled", + "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + } + assert "output_config" not in result + + +def test_effort_high_maps_to_high_budget_for_sonnet_4_5(): + result = _transform("claude-sonnet-4-5", _claude_code_payload(effort="high")) + + assert result["thinking"] == { + "type": "enabled", + "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + } + assert "output_config" not in result + + +def test_adaptive_effort_passes_through_untouched_for_4_6(): + """4.6+ natively supports the adaptive interface, so it must not be rewritten.""" + result = _transform("claude-sonnet-4-6", _claude_code_payload(effort="high")) + + assert result["thinking"] == {"type": "adaptive"} + assert result["output_config"] == {"effort": "high"} + + +def test_thinking_and_effort_dropped_for_non_reasoning_model(): + """A model with no reasoning support cannot take thinking or effort, so both are + silently dropped (no drop_params required) so the request still succeeds.""" + result = _transform("claude-3-5-haiku-latest", _claude_code_payload(effort="medium")) + + assert "thinking" not in result + assert "output_config" not in result + + +def test_residual_output_config_preserved_after_effort_translation(): + """output_config may carry `format` (structured outputs) alongside effort. Only + the consumed effort key is removed; the residual is left for provider subclasses + (bedrock/vertex) to handle, and effort is translated to legacy thinking.""" + result = _transform( + "claude-haiku-4-5", + _claude_code_payload(effort="medium", format={"type": "json_schema"}), + ) + + assert result["thinking"] == { + "type": "enabled", + "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + } + assert result["output_config"] == {"format": {"type": "json_schema"}} + + +def test_opus_4_5_keeps_effort_but_drops_adaptive_thinking(): + """Regression: Opus 4.5 advertises supports_output_config (accepts + output_config.effort) but is NOT adaptive, so thinking:{type:adaptive} is + rejected by Anthropic. The effort must be kept and only the adaptive thinking + block dropped, rather than early-returning and forwarding adaptive thinking raw.""" + result = _transform("claude-opus-4-5", _claude_code_payload(effort="medium")) + + assert result["output_config"] == {"effort": "medium"} + assert "thinking" not in result + + +def test_opus_4_5_preserves_native_effort_without_adaptive_thinking(): + """A caller sending output_config.effort alone (no adaptive thinking) to Opus 4.5 + must pass through untouched, since the model supports it natively.""" + result = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-5", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={ + "max_tokens": 8192, + "output_config": {"effort": "high"}, + }, + litellm_params={}, + headers={}, + ) + + assert result["output_config"] == {"effort": "high"} + assert "thinking" not in result + + +def test_opus_4_5_unsupported_effort_level_translated_to_legacy_thinking(): + """Opus 4.5 accepts output_config.effort but only levels low/medium/high; + Claude Code defaults to xhigh on newer models, and forwarding that level raw + would be rejected with "effort='xhigh' is not supported by this model". An + unsupported level must fall through to the legacy translation (budget-based + thinking, effort stripped) instead of being preserved.""" + result = _transform("claude-opus-4-5", _claude_code_payload(effort="xhigh", max_tokens=64000)) + + assert result["thinking"] == { + "type": "enabled", + "budget_tokens": DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, + } + assert "output_config" not in result + + +def test_opus_4_5_effort_only_unsupported_level_left_for_provider_normalization(): + """An effort-only request (no adaptive thinking) must pass through untouched even + when the level exceeds what the model supports: provider subclasses own their + level normalization (bedrock clamps xhigh to the model's ceiling after this base + transform runs), so consuming the effort here breaks that contract.""" + result = _transform( + "claude-opus-4-5", + {"max_tokens": 4096, "output_config": {"effort": "xhigh"}}, + ) + + assert result["output_config"] == {"effort": "xhigh"} + assert "thinking" not in result + + +def test_budget_capped_below_max_tokens(): + """Adaptive thinking carries no budget, so the translated legacy budget must be + capped below max_tokens (Anthropic requires max_tokens > budget_tokens). A + high-effort budget (4096) with max_tokens=3000 must be capped to 2999.""" + result = _transform("claude-haiku-4-5", _claude_code_payload(effort="high", max_tokens=3000)) + + assert result["thinking"] == {"type": "enabled", "budget_tokens": 2999} + + +def test_thinking_dropped_when_max_tokens_too_small_for_min_budget(): + """When max_tokens can't fit even the minimum thinking budget, thinking is + silently dropped so the request still succeeds rather than being rejected.""" + result = _transform("claude-haiku-4-5", _claude_code_payload(effort="medium", max_tokens=512)) + + assert "thinking" not in result + assert "output_config" not in result + + +def test_unrecognized_effort_raises_clean_400(): + """An unrecognized effort value (e.g. a future Anthropic tier) must surface as a + clean AnthropicError 400, matching _translate_reasoning_effort_to_anthropic, + rather than leaking litellm's internal BadRequestError.""" + with pytest.raises(AnthropicError) as exc_info: + _transform("claude-haiku-4-5", _claude_code_payload(effort="turbo")) + + assert exc_info.value.status_code == 400 + + +def test_non_adaptive_request_without_effort_is_untouched(): + """A non-adaptive model receiving a request with no adaptive interface (no + effort, no adaptive thinking) must pass through untouched.""" + result = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 1024}, + litellm_params={}, + headers={}, + ) + + assert "thinking" not in result + assert "output_config" not in result diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index a2fbb68bbb9..2ec2553d0de 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1579,7 +1579,7 @@ class TestClaudeOpus48AdaptiveThinking: def test_adaptive_thinking_detected_for_opus_4_8(self, local_model_cost_map, model): from litellm.llms.anthropic.common_utils import AnthropicModelInfo - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is True + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True def test_resolver_reads_flag_through_bedrock_invoke_prefix( self, local_model_cost_map @@ -1593,6 +1593,7 @@ class TestClaudeOpus48AdaptiveThinking: AnthropicModelInfo._supports_model_capability( "bedrock/invoke/us.anthropic.claude-opus-4-8", "supports_adaptive_thinking", + "anthropic", ) is True ) @@ -1610,7 +1611,7 @@ class TestClaudeOpus48AdaptiveThinking: def test_adaptive_thinking_detected_for_fable_5(self, local_model_cost_map, model): from litellm.llms.anthropic.common_utils import AnthropicModelInfo - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is True + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True @pytest.mark.parametrize( "model", @@ -1645,27 +1646,28 @@ class TestClaudeOpus48AdaptiveThinking: version (``4.6`` -> ``4-6``).""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is True + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True @pytest.mark.parametrize( "model", [ - "us.anthropic.claude-fable-5-preview", - "claude-fable-5-preview", + "us.anthropic.claude-fable-preview", + "claude-fable-preview", ], ) def test_unmapped_aliases_without_parseable_version_stay_non_adaptive( self, local_model_cost_map, model ): """An alias absent from the map, not matched by any ``fallback_generalizations`` - rule, and without a parseable opus/sonnet/haiku >= 4.6 family version stays - non-adaptive. ``fable`` is outside the version-rule family set, so neither the - cost map nor the declarative rule marks it adaptive.""" + rule, and without any parseable family version stays non-adaptive. ``fable`` + without a major version matches neither the core-family 4.6+ gate nor the + family-agnostic 5+ gate, so neither the cost map nor the declarative rule marks + it adaptive.""" import litellm from litellm.llms.anthropic.common_utils import AnthropicModelInfo assert model not in litellm.model_cost - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is False + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is False @pytest.mark.parametrize( "model", @@ -1677,21 +1679,23 @@ class TestClaudeOpus48AdaptiveThinking: "claude-opus-5-0", "claude-opus-4-10", "claude-opus-4-8-some-future-suffix", + "claude-fable-5-preview", + "us.anthropic.claude-fable-5-preview", ], ) def test_adaptive_thinking_version_fallback_for_unmapped_high_versions( self, local_model_cost_map, model ): - """Provider-prefixed or suffixed Claude names that resolve to no mapped entry and - are not matched by the anchored ``anthropic-claude`` pricing rule still resolve to - adaptive when their opus/sonnet/haiku family version is >= 4.6. The version gate is - the declarative ``anthropic-claude-adaptive-thinking`` rule, so 5.x, 6.x and any - later family are covered with no code change.""" + """Provider-prefixed or suffixed Claude names that resolve to no mapped entry + still resolve to adaptive when the id carries claude-- at version 4.6 + or higher, bare 5+ majors included. The version gate is the declarative + ``claude-adaptive-thinking`` rule, so 5.x, 6.x and any later family are covered + with no code change.""" import litellm from litellm.llms.anthropic.common_utils import AnthropicModelInfo assert model not in litellm.model_cost - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is True + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True @pytest.mark.parametrize( "model", @@ -1714,7 +1718,7 @@ class TestClaudeOpus48AdaptiveThinking: from litellm.llms.anthropic.common_utils import AnthropicModelInfo assert model not in litellm.model_cost - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is False + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is False @pytest.mark.parametrize( "model", @@ -1723,4 +1727,52 @@ class TestClaudeOpus48AdaptiveThinking: def test_non_adaptive_models_not_detected(self, local_model_cost_map, model): from litellm.llms.anthropic.common_utils import AnthropicModelInfo - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is False + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is False + + +class TestCapabilityProbeUsesCallerProvider: + """``_supports_model_capability`` must probe under the caller's real provider + namespace instead of a pinned ``"anthropic"``. With the pin, the exact Bedrock + cost-map entry for ``global.anthropic.claude-opus-4-8`` was rejected by the + provider match and the anthropic-scoped fallback rule answered instead, so + flipping ``supports_adaptive_thinking`` on the exact entry changed nothing and + the documented "exact entry beats rule" precedence was silently violated.""" + + BEDROCK_MODEL = "global.anthropic.claude-opus-4-8" + + def test_exact_bedrock_entry_flag_is_authoritative_for_bedrock_caller( + self, local_model_cost_map, monkeypatch + ): + import litellm + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + assert ( + AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock") + is True + ) + + monkeypatch.setitem( + litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False + ) + litellm.get_model_info.cache_clear() + + assert ( + AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock") + is False + ) + + def test_native_anthropic_probe_still_reads_anthropic_entry( + self, local_model_cost_map, monkeypatch + ): + import litellm + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + monkeypatch.setitem( + litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False + ) + litellm.get_model_info.cache_clear() + + assert ( + AnthropicModelInfo._is_adaptive_thinking_model("claude-opus-4-8", "anthropic") + is True + ) diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 5983597196a..5e9af6bd34d 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -331,3 +331,59 @@ class TestProviderConfigManagerAzureAnthropicMessages: ) assert config is None + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Force the bundled backup cost map so capability flags match this branch.""" + import litellm + + original = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original + litellm.get_model_info.cache_clear() + + +def test_messages_thinking_shape_follows_exact_azure_entry_flag(local_model_cost_map, monkeypatch): + """The Azure messages config must probe capabilities under ``azure_ai`` so an + operator setting ``supports_adaptive_thinking: false`` on the exact + ``azure_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry. + With the inherited ``"anthropic"`` provider default the flip was ignored and + the transform kept emitting ``thinking.type='adaptive'``.""" + import litellm + + config = AzureAnthropicMessagesConfig() + + def transform(): + return config.transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={ + "max_tokens": 4096, + "reasoning_effort": "medium", + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + result = transform() + assert result.get("thinking") == {"type": "adaptive"} + assert result.get("output_config") == {"effort": "medium"} + + monkeypatch.setitem( + litellm.model_cost["azure_ai/claude-opus-4-8"], "supports_adaptive_thinking", False + ) + litellm.get_model_info.cache_clear() + assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True + + flipped = transform() + thinking = flipped.get("thinking") + assert isinstance(thinking, dict) + assert thinking.get("type") == "enabled" + assert isinstance(thinking.get("budget_tokens"), int) + assert "output_config" not in flipped diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 03d0d87a58c..2293aaed60c 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -1893,6 +1893,217 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system(): ] +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-opus-4-8", + "jp.anthropic.claude-opus-4-8", + "us.anthropic.claude-sonnet-5", + "us.anthropic.claude-fable-5", + ], +) +def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map, model): + """Regression test for the Bedrock prompt-cache collapse: hoisting a + mid-conversation ``role: "system"`` message (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) into the top-level + ``system`` field mutates the cache prefix and invalidates the cached message + history, so on models flagged ``supports_mid_conversation_system`` (Claude + 4.8+, which Invoke accepts the role on) such entries must be forwarded + in place. Billing-header blocks must still be stripped from the top-level + ``system`` field even when nothing is hoisted.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model=model, + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={ + "max_tokens": 256, + "stream": False, + "system": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=2.1.205;"}, + {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}, + ], + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == messages + assert result["system"] == [ + {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}} + ] + + +def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cost_map): + """On models flagged ``supports_mid_conversation_system``, only the leading + run of ``role: "system"`` messages is hoisted into the top-level ``system`` + field; a later system entry keeps its position in ``messages`` so the + serialized prefix stays stable across turns.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": "Cite sources."}, + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-8", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "You are terse."}, + {"type": "text", "text": "Cite sources."}, + ] + + +def test_bedrock_invoke_transform_hoists_mid_conversation_system_for_older_claude(local_model_cost_map): + """Regression test for Claude Code 400s on pre-Opus-4.8 Bedrock models: + Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet 4.6, + Haiku 4.5, etc. ("role 'system' is not supported on this model"), so on + models without ``supports_mid_conversation_system`` every system entry must + be hoisted into the top-level ``system`` field, mid-conversation ones + included.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-opus-4-7", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={ + "max_tokens": 256, + "stream": False, + "system": [{"type": "text", "text": "Base."}], + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "read the file"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "Base."}, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ] + + +def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_model_cost_map): + """A model with no cost-map entry and no fallback-generalization rule gets + the hoist-everything behavior: the safe default is a mutated cache prefix, + never a provider 400 from forwarding a role the model may not accept.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-opus-3-9", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [{"type": "text", "text": "mid-conversation reminder"}] + + +def test_bedrock_invoke_transform_keeps_system_in_place_for_unmapped_future_claude(local_model_cost_map): + """An unmapped Bedrock Claude at 4.8 or higher resolves through the + ``claude-mid-conversation-system`` capability rule, so a future model that + has not landed in the cost map yet keeps the cache-preserving in-place + behavior instead of falling back to hoist-all.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-opus-4-9", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == messages + assert "system" not in result + + +def test_bedrock_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): + """Exact cost-map hits resolve before fallback-generalization rules, so a + mapped Bedrock Claude 4.8+ entry without ``supports_mid_conversation_system`` + silently loses the cache-preserving in-place handling that the + ``claude-mid-conversation-system`` capability rule grants unmapped ids. + Every mapped bedrock entry the rule's own pattern matches must carry the + flag explicitly.""" + import re + + import litellm + + cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") + with open(cost_map_path) as f: + cost_map = json.load(f) + rules = cost_map["fallback_generalizations"]["rules"] + pattern = re.compile( + next(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"), + re.IGNORECASE, + ) + missing = [ + key + for key, info in cost_map.items() + if isinstance(info, dict) + and str(info.get("litellm_provider", "")).startswith("bedrock") + and pattern.search(key) + and info.get("supports_mid_conversation_system") is not True + ] + assert missing == [] + + def test_as_system_content_blocks_handles_each_shape(): """``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty, a string -> a single text block, a list -> a shallow copy, and any other value @@ -2009,3 +2220,46 @@ def test_bedrock_clear_thinking_leaves_enabled_thinking_on_non_adaptive_model(): assert changed is False assert request["thinking"] == {"type": "enabled", "budget_tokens": 8000} assert "output_config" not in request + + +def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag( + local_model_cost_map, monkeypatch +): + """The outbound thinking payload must follow the exact Bedrock cost-map entry. + Before threading the caller's provider through the capability probes, the probe + was pinned to ``"anthropic"``: the exact ``global.anthropic.claude-opus-4-8`` + entry was rejected by the provider match and the anthropic-scoped fallback rule + forced ``thinking.type='adaptive'`` even with ``supports_adaptive_thinking`` + explicitly set to ``false`` on the entry.""" + import litellm + + from litellm.types.router import GenericLiteLLMParams + + model = "global.anthropic.claude-opus-4-8" + cfg = AmazonAnthropicClaudeMessagesConfig() + + def transform(): + return cfg.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + anthropic_messages_optional_request_params={ + "max_tokens": 4096, + "reasoning_effort": "medium", + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + result = transform() + assert result.get("thinking") == {"type": "adaptive"} + assert result.get("output_config") == {"effort": "medium"} + + monkeypatch.setitem(litellm.model_cost[model], "supports_adaptive_thinking", False) + litellm.get_model_info.cache_clear() + + flipped = transform() + thinking = flipped.get("thinking") + assert isinstance(thinking, dict) + assert thinking.get("type") == "enabled" + assert isinstance(thinking.get("budget_tokens"), int) + assert "output_config" not in flipped diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index cfdb76a97f4..00f3e7a6faf 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -416,3 +416,10 @@ def test_transform_request_keeps_parallel_tool_calls_for_claude(): )["messages"] assert len([m for m in result if m.get("role") == "assistant"]) == 1 + + +def test_databricks_config_probes_capabilities_under_databricks_namespace(): + """Inherited AnthropicConfig capability probes read ``self.custom_llm_provider``; + without this override they probed the ``anthropic`` cost-map namespace and + ignored the exact ``databricks/databricks-claude-*`` entries.""" + assert DatabricksConfig().custom_llm_provider == "databricks" diff --git a/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py b/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py index 01787c07d27..8ed84b3ed8d 100644 --- a/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py +++ b/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py @@ -326,3 +326,11 @@ def test_github_copilot_config_does_not_handle_web_search_natively(): assert GithubCopilotAnthropicMessagesConfig().handles_web_search_natively() is False assert AnthropicMessagesConfig().handles_web_search_natively() is True + + +def test_github_copilot_messages_config_probes_capabilities_under_copilot_namespace(): + """Capability probes in the shared pass-through helpers read + ``self.custom_llm_provider``; without this override they probed the + ``anthropic`` namespace and ignored the exact ``github_copilot/claude-*`` + cost-map entries.""" + assert GithubCopilotAnthropicMessagesConfig().custom_llm_provider == "github_copilot" diff --git a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index 534d7aefda4..33e677b000e 100644 --- a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -299,3 +299,21 @@ def test_anthropic_beta_survives_provider_filter_on_passthrough_path(config): stripped = update_headers_with_filtered_beta(headers=dict(headers), provider="openai") assert "anthropic-beta" not in stripped + + +def test_json_provider_messages_config_probes_capabilities_under_provider_slug(): + """Capability probes in the shared pass-through helpers read + ``self.custom_llm_provider``. The JSON-provider config knows its slug, so it + must expose it; the generic OpenAI-like config has no class-level namespace + and keeps the inherited ``anthropic`` default.""" + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + provider = SimpleProviderConfig( + slug="exampleprovider", + data={"base_url": "https://api.example.com/v1", "api_key_env": "EXAMPLE_API_KEY"}, + ) + assert JSONProviderAnthropicMessagesConfig(provider).custom_llm_provider == "exampleprovider" + assert OpenAILikeAnthropicMessagesConfig().custom_llm_provider == "anthropic" diff --git a/tests/test_litellm/llms/openai_like/test_meta_provider.py b/tests/test_litellm/llms/openai_like/test_meta_provider.py new file mode 100644 index 00000000000..ec43f54dab7 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_meta_provider.py @@ -0,0 +1,242 @@ +""" +Tests for the Meta Model API (Muse Spark) provider configuration and integration. +""" + +import pytest + +import litellm + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Force the bundled cost map so muse-spark-1.1 resolves. + + muse-spark-1.1 is a day-0 model that ships only in the bundled backup; the + default remote fetch of ``main`` does not carry it yet, so tests that read its + model info must pin the local map instead of depending on network state or a + leaked cost map from an earlier test. + """ + original_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_cost + litellm.get_model_info.cache_clear() + + +class TestMetaProviderConfig: + def test_meta_in_provider_list(self): + from litellm import LlmProviders + + assert hasattr(LlmProviders, "META") + assert LlmProviders.META.value == "meta" + assert "meta" in litellm.provider_list + + def test_meta_json_config_exists(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("meta") + + meta = JSONProviderRegistry.get("meta") + assert meta is not None + assert meta.base_url == "https://api.meta.ai/v1" + assert meta.api_key_env == "META_API_KEY" + assert meta.api_base_env == "META_API_BASE" + + def test_meta_supports_responses_api(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.supports_responses_api("meta") + + def test_meta_in_openai_compatible_providers(self): + from litellm.constants import openai_compatible_providers + + assert "meta" in openai_compatible_providers + + def test_meta_provider_resolution(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="meta/muse-spark-1.1", + custom_llm_provider=None, + api_base=None, + api_key="sk-test", + ) + + assert model == "muse-spark-1.1" + assert provider == "meta" + assert api_base == "https://api.meta.ai/v1" + + def test_meta_api_base_override(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="meta/muse-spark-1.1", + custom_llm_provider=None, + api_base="https://custom.meta.ai/v1", + api_key="sk-test", + ) + + assert provider == "meta" + assert api_base == "https://custom.meta.ai/v1" + assert api_key == "sk-test" + + def test_meta_url_autodetection(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="muse-spark-1.1", + custom_llm_provider=None, + api_base="https://api.meta.ai/v1", + api_key=None, + ) + assert provider == "meta" + assert api_base == "https://api.meta.ai/v1" + + def test_meta_router_config(self): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "muse-spark", + "litellm_params": { + "model": "meta/muse-spark-1.1", + "api_key": "test-key", + }, + } + ] + ) + + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "muse-spark" + + +class TestMetaReasoningParams: + def test_muse_spark_supports_reasoning_effort(self, local_model_cost_map): + params = litellm.get_supported_openai_params(model="muse-spark-1.1", custom_llm_provider="meta") + assert params is not None + assert "reasoning_effort" in params + + def test_reasoning_effort_mapped_through(self, local_model_cost_map): + cfg = litellm.ProviderConfigManager.get_provider_chat_config( + model="muse-spark-1.1", provider=litellm.LlmProviders.META + ) + assert cfg is not None + mapped = cfg.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model="muse-spark-1.1", + drop_params=False, + ) + assert mapped["reasoning_effort"] == "xhigh" + + def test_reasoning_effort_gated_on_capability(self): + """A meta model without reasoning metadata must not advertise reasoning_effort.""" + params = litellm.get_supported_openai_params(model="some-non-reasoning-model", custom_llm_provider="meta") + assert params is not None + assert "reasoning_effort" not in params + + +class TestMetaAnthropicMessages: + def test_meta_resolves_native_messages_config(self): + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + cfg = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="muse-spark-1.1", provider=litellm.LlmProviders.META + ) + assert isinstance(cfg, JSONProviderAnthropicMessagesConfig) + + def test_json_provider_without_messages_endpoint_resolves_none(self): + cfg = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="some-model", provider=litellm.LlmProviders.PINSTRIPES + ) + assert cfg is None + + def test_complete_url_defaults_to_meta_base(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + provider = JSONProviderRegistry.get("meta") + assert provider is not None + cfg = JSONProviderAnthropicMessagesConfig(provider) + + url = cfg.get_complete_url( + api_base=None, + api_key="sk-test", + model="muse-spark-1.1", + optional_params={}, + litellm_params={}, + ) + assert url == "https://api.meta.ai/v1/messages" + + override_url = cfg.get_complete_url( + api_base="https://custom.meta.ai/v1", + api_key="sk-test", + model="muse-spark-1.1", + optional_params={}, + litellm_params={}, + ) + assert override_url == "https://custom.meta.ai/v1/messages" + + def test_api_key_resolved_from_env(self, monkeypatch): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + monkeypatch.setenv("META_API_KEY", "sk-env-key") + provider = JSONProviderRegistry.get("meta") + assert provider is not None + cfg = JSONProviderAnthropicMessagesConfig(provider) + + headers, _ = cfg.validate_anthropic_messages_environment( + headers={}, + model="muse-spark-1.1", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + assert headers["authorization"] == "Bearer sk-env-key" + assert headers["anthropic-version"] == "2023-06-01" + + +class TestMuseSparkModelInfo: + def test_muse_spark_pricing_and_capabilities(self, local_model_cost_map): + info = litellm.get_model_info("meta/muse-spark-1.1") + + assert info["litellm_provider"] == "meta" + assert info["input_cost_per_token"] == 1.25e-06 + assert info["output_cost_per_token"] == 4.25e-06 + assert info["cache_read_input_token_cost"] == 1.5e-07 + assert info["max_input_tokens"] == 1048576 + assert info["supports_reasoning"] is True + assert info["supports_web_search"] is True + assert info["supports_vision"] is True + assert info["supports_function_calling"] is True + assert info["supports_prompt_caching"] is True + + def test_muse_spark_cost_calculation(self, local_model_cost_map): + from litellm import completion_cost + from litellm.types.utils import ModelResponse, Usage + + response = ModelResponse( + model="muse-spark-1.1", + usage=Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500), + ) + cost = completion_cost( + completion_response=response, + model="meta/muse-spark-1.1", + custom_llm_provider="meta", + ) + expected = 1000 * 1.25e-06 + 500 * 4.25e-06 + assert abs(cost - expected) < 1e-12 diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index bfd37f73b2d..ce770221ceb 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -509,3 +509,59 @@ def test_vertex_claude_completion_does_not_mutate_shared_extra_headers(): assert ( shared_extra_headers == {} ), "extra_headers must not be mutated by completion()" + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Force the bundled backup cost map so capability flags match this branch.""" + import litellm + + original = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original + litellm.get_model_info.cache_clear() + + +def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cost_map, monkeypatch): + """The Vertex messages config must probe capabilities under ``vertex_ai`` so an + operator setting ``supports_adaptive_thinking: false`` on the exact + ``vertex_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry. + With the inherited ``"anthropic"`` provider default the flip was ignored and + the transform kept emitting ``thinking.type='adaptive'``.""" + import litellm + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + + def transform(): + return config.transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={ + "max_tokens": 4096, + "reasoning_effort": "medium", + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + result = transform() + assert result.get("thinking") == {"type": "adaptive"} + assert result.get("output_config") == {"effort": "medium"} + + monkeypatch.setitem( + litellm.model_cost["vertex_ai/claude-opus-4-8"], "supports_adaptive_thinking", False + ) + litellm.get_model_info.cache_clear() + assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True + + flipped = transform() + thinking = flipped.get("thinking") + assert isinstance(thinking, dict) + assert thinking.get("type") == "enabled" + assert isinstance(thinking.get("budget_tokens"), int) + assert "output_config" not in flipped diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aim.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aim.py new file mode 100644 index 00000000000..2e83422074e --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aim.py @@ -0,0 +1,88 @@ +"""Tests for the AIM guardrail's inspection-payload construction.""" + +from litellm.proxy.guardrails.guardrail_hooks.aim.aim import AimGuardrail + + +def test_aim_inspection_messages_coerces_chat_completions_tool_role_to_user(): + """LIT-4294: A valid chat-completions ``role: "tool"`` message carries a + ``tool_call_id``, but the inspection flatten drops every field except + ``role`` and ``content``. A bare ``tool`` message without ``tool_call_id`` + is schema-invalid per the OpenAI chat schema, and the customer's writeup + reproduced AIM's ``/fw/v1/analyze`` returning 422 on exactly that shape. + The AIM POST collapses the role to ``user``; the outbound request to the + LLM is untouched.""" + data = { + "messages": [ + {"role": "user", "content": "weather in SF"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "user", "content": "weather in SF"}, + {"role": "user", "content": "sunny"}, + ] + + +def test_aim_inspection_messages_coerces_non_standard_caller_role_to_user(): + """LIT-4294: A caller-supplied role outside {system, user, assistant} + (e.g. ``developer``, ``function``) is coerced to ``user`` for the AIM + POST, since AIM validates the payload against the OpenAI chat schema + and rejects unknown roles the same way it rejects bare ``tool``.""" + data = { + "messages": [ + {"role": "developer", "content": "system-ish instruction"}, + {"role": "user", "content": "normal user text"}, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "user", "content": "system-ish instruction"}, + {"role": "user", "content": "normal user text"}, + ] + + +def test_aim_inspection_messages_coerces_responses_function_call_output_role(): + """LIT-4294: the shared helper synthesises ``role: "tool"`` for a + Responses ``function_call_output`` item (semantic equivalent of + chat-completions tool messages). AIM's schema-validating POST cannot + carry ``tool_call_id`` in the flat inspection payload, so AIM collapses + that ``tool`` role to ``user`` locally before POSTing.""" + data = { + "input": [ + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "sunny"}], + }, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "user", "content": "sunny"}, + ] + + +def test_aim_inspection_messages_preserves_safe_roles(): + """Safe roles pass through untouched — the coercion only fires for + roles the OpenAI chat schema flatten cannot represent standalone.""" + data = { + "messages": [ + {"role": "system", "content": "be helpful"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "system", "content": "be helpful"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] diff --git a/tests/test_litellm/proxy/guardrails/test_content_utils.py b/tests/test_litellm/proxy/guardrails/test_content_utils.py index 099fca78a62..34d92505359 100644 --- a/tests/test_litellm/proxy/guardrails/test_content_utils.py +++ b/tests/test_litellm/proxy/guardrails/test_content_utils.py @@ -8,7 +8,6 @@ from litellm.proxy.guardrails._content_utils import ( walk_user_text, ) - # ── iter_message_text ──────────────────────────────────────────────────────────── @@ -101,6 +100,55 @@ def test_iter_message_text_empty_data(): assert list(iter_message_text({"input": ""})) == [] +def test_iter_message_text_responses_api_input_text_and_output_text_parts(): + """LIT-4294: Responses-API content parts use ``input_text`` (request) and + ``output_text`` (assistant); reading only ``type == "text"`` skipped every + ``/v1/responses`` body and every text guardrail was a no-op on that path.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "user text"}], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "assistant text"}], + }, + ] + } + assert list(iter_message_text(data)) == ["user text", "assistant text"] + + +def test_iter_message_text_responses_api_tool_call_taxonomy(): + """LIT-4294: a Responses ``input`` list freely mixes message items, + ``function_call`` (no ``role``), and ``function_call_output`` items. The + old ``all(item has 'role')`` gate wrapped the whole list as one blob and + yielded nothing; every text fragment must be visited independently.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "sunny"}], + }, + ] + } + assert list(iter_message_text(data)) == ["hello", "sunny"] + + # ── walk_user_text ──────────────────────────────────────────────────────────── @@ -160,6 +208,89 @@ def test_walk_user_text_redacts_responses_api_list_input(): assert data["input"][1] == {"type": "image_url", "image_url": {"url": "..."}} +def test_walk_user_text_redacts_responses_input_text_and_output_text_parts(): + """LIT-4294: ``walk_user_text`` must recognise the Responses text-part + variants so masking guardrails (secret detection, PII) actually redact + ``/v1/responses`` bodies instead of no-op'ing on them.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "AKIAEXAMPLE"}], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "AKIAEXAMPLE too"}], + }, + ] + } + visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]")) + assert visited == 2 + assert data["input"][0]["content"][0] == { + "type": "input_text", + "text": "[REDACTED]", + } + assert data["input"][1]["content"][0] == { + "type": "output_text", + "text": "[REDACTED] too", + } + + +def test_walk_user_text_redacts_function_call_output_text(): + """LIT-4294: tool-call round-trips carry secrets in + ``function_call_output.output``; the redact walker must descend into it + while leaving ``function_call`` items (call_id, arguments) untouched.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "AKIAEXAMPLE user"}], + }, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": '{"AKIAEXAMPLE": 1}', + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "AKIAEXAMPLE tool"}], + }, + ] + } + visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]")) + assert visited == 2 + assert data["input"][0]["content"][0]["text"] == "[REDACTED] user" + assert data["input"][1] == { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": '{"AKIAEXAMPLE": 1}', + } + assert data["input"][2]["output"][0]["text"] == "[REDACTED] tool" + + +def test_walk_user_text_redacts_function_call_output_string_output(): + """LIT-4294: ``function_call_output.output`` is also a plain string in + OpenAI's Responses spec; the redact walker must handle both forms.""" + data = { + "input": [ + { + "type": "function_call_output", + "call_id": "c1", + "output": "AKIAEXAMPLE tool", + }, + ] + } + visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]")) + assert visited == 1 + assert data["input"][0]["output"] == "[REDACTED] tool" + + def test_walk_user_text_redacts_mixed_list_input(): """Read and write helpers must agree on coverage — bare strings inside a mixed ``input`` list are inspected by both.""" @@ -206,17 +337,13 @@ def test_build_inspection_messages_joins_multimodal_text_parts(): } ] } - assert build_inspection_messages(data) == [ - {"role": "user", "content": "first part\nsecond part"} - ] + assert build_inspection_messages(data) == [{"role": "user", "content": "first part\nsecond part"}] def test_build_inspection_messages_lifts_responses_api_input(): """fniVO9-F: ``input`` must be visible to hooks that POST messages to a remote API.""" data = {"input": "responses-api content"} - assert build_inspection_messages(data) == [ - {"role": "user", "content": "responses-api content"} - ] + assert build_inspection_messages(data) == [{"role": "user", "content": "responses-api content"}] def test_build_inspection_messages_drops_messages_with_no_text(): @@ -233,6 +360,102 @@ def test_build_inspection_messages_drops_messages_with_no_text(): assert build_inspection_messages(data) == [{"role": "user", "content": "kept"}] +def test_build_inspection_messages_responses_api_tool_call_taxonomy(): + """LIT-4294: mixed Responses ``input`` (message + function_call + + function_call_output) must produce a non-empty inspection list. The + customer's writeup reproduced a 422 from AIM's ``/fw/v1/analyze`` + (``No messages in the request``) when this synthesised list came back + empty; every other guardrail silently scanned nothing on the same + input.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "sunny"}], + }, + ] + } + assert build_inspection_messages(data) == [ + {"role": "user", "content": "hello"}, + {"role": "tool", "content": "sunny"}, + ] + + +def test_build_inspection_messages_function_call_output_defaults_to_tool(): + """LIT-4294: a Responses ``function_call_output`` item is the semantic + equivalent of a chat-completions ``role: "tool"`` message, so the shared + helper synthesises ``role: "tool"`` when the item has no explicit role. + AIM's schema-safe coercion happens at the AIM call site, not here.""" + data = { + "input": [ + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "tool text"}], + }, + ] + } + assert build_inspection_messages(data) == [{"role": "tool", "content": "tool text"}] + + +def test_build_inspection_messages_function_call_output_preserves_explicit_role(): + """When ``function_call_output`` carries a caller-supplied ``role`` the + shared helper preserves it rather than synthesising ``tool``.""" + data = { + "input": [ + { + "type": "function_call_output", + "role": "assistant", + "call_id": "c1", + "output": [{"type": "input_text", "text": "tool text"}], + }, + ] + } + assert build_inspection_messages(data) == [{"role": "assistant", "content": "tool text"}] + + +def test_build_inspection_messages_bare_content_part_preserves_explicit_role(): + """A bare content-part dict with an explicit ``role`` keeps it. Only + absent roles get defaulted to ``user``.""" + data = { + "input": [ + {"type": "input_text", "text": "no role"}, + {"type": "output_text", "role": "assistant", "text": "with role"}, + ] + } + assert build_inspection_messages(data) == [ + {"role": "user", "content": "no role"}, + {"role": "assistant", "content": "with role"}, + ] + + +def test_build_inspection_messages_message_item_preserves_role(): + """Responses message items carry a role explicitly; the shared helper + passes it through untouched.""" + data = { + "input": [ + {"type": "message", "role": "system", "content": [{"type": "input_text", "text": "sys"}]}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "asst"}]}, + ] + } + assert build_inspection_messages(data) == [ + {"role": "system", "content": "sys"}, + {"role": "assistant", "content": "asst"}, + ] + + def test_build_inspection_messages_empty_data(): assert build_inspection_messages({}) == [] assert build_inspection_messages({"messages": []}) == [] diff --git a/tests/test_litellm/test_claude_fable_5_config.py b/tests/test_litellm/test_claude_fable_5_config.py index d8d95fba0da..3a9ebf65bbb 100644 --- a/tests/test_litellm/test_claude_fable_5_config.py +++ b/tests/test_litellm/test_claude_fable_5_config.py @@ -204,7 +204,7 @@ def test_adaptive_thinking_detected_for_fable_5(local_model_cost_map, model): maps to ``thinking.type='adaptive'`` + ``output_config.effort``.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo - assert AnthropicModelInfo._is_adaptive_thinking_model(model) is True + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True @pytest.mark.parametrize( diff --git a/tests/test_litellm/test_muse_spark_1_1_model_metadata.py b/tests/test_litellm/test_muse_spark_1_1_model_metadata.py new file mode 100644 index 00000000000..540b97884dc --- /dev/null +++ b/tests/test_litellm/test_muse_spark_1_1_model_metadata.py @@ -0,0 +1,63 @@ +import json +from pathlib import Path + +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +MUSE_SPARK_MODEL = "meta/muse-spark-1.1" + + +def test_muse_spark_1_1_model_info(): + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + info = model_cost.get(MUSE_SPARK_MODEL) + assert info is not None, f"{MUSE_SPARK_MODEL} not found in model_prices_and_context_window.json" + + assert info["litellm_provider"] == "meta" + assert info["mode"] == "chat" + + assert info["input_cost_per_token"] == 1.25e-06 + assert info["output_cost_per_token"] == 4.25e-06 + assert info["cache_read_input_token_cost"] == 1.5e-07 + + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 131072 + assert info["max_tokens"] == 131072 + + assert info["supports_function_calling"] is True + assert info["supports_parallel_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert info["supports_tool_choice"] is True + assert info["supports_vision"] is True + assert info["supports_pdf_input"] is True + assert info["supports_web_search"] is True + assert info["supports_minimal_reasoning_effort"] is True + assert info["supports_xhigh_reasoning_effort"] is True + + assert info["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + assert info["supported_modalities"] == ["text", "image", "video"] + assert info["supported_output_modalities"] == ["text"] + + routed_model, provider, _, api_base = get_llm_provider(model=MUSE_SPARK_MODEL, api_key="sk-test") + assert routed_model == "muse-spark-1.1" + assert provider == "meta" + assert api_base == "https://api.meta.ai/v1" + + +def test_muse_spark_1_1_backup_matches_main(): + """Ensure the bundled model cost map stays in sync with the canonical file.""" + repo_root = Path(__file__).parents[2] + main_path = repo_root / "model_prices_and_context_window.json" + backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json" + + with open(main_path) as f: + main_cost = json.load(f) + with open(backup_path) as f: + backup_cost = json.load(f) + + assert backup_cost.get(MUSE_SPARK_MODEL) == main_cost.get(MUSE_SPARK_MODEL), ( + f"{MUSE_SPARK_MODEL} differs between main and backup model cost maps" + ) diff --git a/tests/test_litellm/test_register_model_custom_pricing.py b/tests/test_litellm/test_register_model_custom_pricing.py index e384d3e1161..08f6f76b815 100644 --- a/tests/test_litellm/test_register_model_custom_pricing.py +++ b/tests/test_litellm/test_register_model_custom_pricing.py @@ -303,8 +303,9 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(): """Registering a custom override under a key shape that - ``get_model_info`` cannot resolve (e.g. a double provider prefix like - ``bedrock/bedrock/us.anthropic.claude-sonnet-4-6``) must still inherit + ``get_model_info`` cannot resolve (e.g. a triple provider prefix like + ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6``; a double + prefix now resolves like a routing prefix) must still inherit the built-in cache pricing for the underlying model. Before the fix ``register_model`` fell back to an empty ``existing_model`` @@ -324,7 +325,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(): litellm.model_cost = litellm.get_model_cost_map(url="") builtin_key = "us.anthropic.claude-sonnet-4-6" - registered_key = f"bedrock/bedrock/{builtin_key}" + registered_key = f"bedrock/bedrock/bedrock/{builtin_key}" builtin = litellm.model_cost[builtin_key] assert builtin["cache_creation_input_token_cost"] > 0 diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index c35fb2fcbe2..826e963be26 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -852,6 +852,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, + "supports_mid_conversation_system": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, "supports_speed": {"type": "boolean"}, @@ -871,6 +872,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/embeddings", "/v1/chat/completions", "/v1/completions", + "/v1/messages", "/v1/images/generations", "/v1/realtime", "/v1/realtime/transcription_sessions", @@ -881,7 +883,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/audio/speech", "/v1/ocr", "/vertex_ai/live", - "/v1/realtime/transcription_sessions", ], }, }, @@ -1063,6 +1064,43 @@ def test_get_model_info_gemini(): assert info.get("rpm") is not None, f"{model} does not have rpm" +def test_get_model_info_bedrock_regional_inference_profile_pricing(local_model_cost_map): + """Regression LIT-4056: with the bedrock/ routing prefix (plain, converse/, or + invoke/), the exact regional cost-map entry must win over the region-stripped + base entry, matching the unprefixed control form.""" + regional = litellm.model_cost["au.anthropic.claude-opus-4-8"] + base = litellm.model_cost["anthropic.claude-opus-4-8"] + assert regional["input_cost_per_token"] > base["input_cost_per_token"] + + for model in ( + "bedrock/au.anthropic.claude-opus-4-8", + "bedrock/converse/au.anthropic.claude-opus-4-8", + "bedrock/invoke/au.anthropic.claude-opus-4-8", + ): + info = litellm.get_model_info(model=model) + assert info["key"] == "au.anthropic.claude-opus-4-8", model + assert info["input_cost_per_token"] == regional["input_cost_per_token"], model + assert info["output_cost_per_token"] == regional["output_cost_per_token"], model + + control = litellm.get_model_info(model="au.anthropic.claude-opus-4-8", custom_llm_provider="bedrock") + assert control["key"] == "au.anthropic.claude-opus-4-8" + + +def test_get_model_info_bedrock_regional_profile_without_entry_falls_back_to_base(local_model_cost_map): + """A regional profile with no dedicated cost-map entry must still resolve to its + region-stripped base entry.""" + assert "jp.anthropic.claude-opus-4-8" not in litellm.model_cost + info = litellm.get_model_info(model="bedrock/jp.anthropic.claude-opus-4-8") + assert info["key"] == "anthropic.claude-opus-4-8" + + +def test_get_model_info_bedrock_double_provider_prefix_resolves(local_model_cost_map): + """A doubled bedrock/ prefix routes at runtime via strip_bedrock_routing_prefix, + so model info must resolve it to the same entry the request actually bills as.""" + info = litellm.get_model_info(model="bedrock/bedrock/us.anthropic.claude-sonnet-4-6") + assert info["key"] == "us.anthropic.claude-sonnet-4-6" + + def test_openai_models_in_model_info(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="")