mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
feat(otel): trace auto-router configuration and classifier failures (#44926)
This commit is contained in:
parent
2477635213
commit
a193c67347
19 changed files with 1031 additions and 73 deletions
|
|
@ -32,6 +32,7 @@ from litellm.integrations.otel.model.db_endpoint import db_span_attributes
|
|||
from litellm.integrations.otel.model.metadata import flatten_metadata
|
||||
from litellm.integrations.otel.model.semconv import LiteLLM, Metric
|
||||
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
|
||||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
|
|
@ -2407,6 +2408,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
metadata: Final = standard_logging_payload["metadata"]
|
||||
for key, value in metadata.items():
|
||||
self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value)
|
||||
decision: Final = metadata.get("routing_decision")
|
||||
span.set_attributes(routing_decision_attributes(decision))
|
||||
|
||||
# get hidden params
|
||||
hidden_params: Final = getattr(standard_logging_payload, "hidden_params", None) or (
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ from litellm.integrations.otel.model.spans import SpanRole, span_role_for_servic
|
|||
from litellm.integrations.otel.model.utils import to_ns
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
active_phase,
|
||||
active_phase_span,
|
||||
is_recordable_span,
|
||||
mcp_message_transport_span,
|
||||
post_response_root,
|
||||
|
|
@ -778,6 +779,11 @@ class OpenTelemetryV2(CustomLogger):
|
|||
if is_recordable_span(span):
|
||||
span.add_event(name, attributes)
|
||||
|
||||
def set_phase_attributes(self, attributes: Mapping[str, str | int | float | bool]) -> None:
|
||||
span: Final = active_phase_span()
|
||||
if span is not None and is_recordable_span(span):
|
||||
span.set_attributes(attributes)
|
||||
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
|
|
@ -1042,6 +1048,12 @@ def phase_event(name: str, attributes: Mapping[str, str | int] | None = None) ->
|
|||
logger.add_phase_event(name, attributes)
|
||||
|
||||
|
||||
def phase_attributes(attributes: Mapping[str, str | int | float | bool]) -> None:
|
||||
logger: Final = _registered_v2_logger()
|
||||
if logger is not None:
|
||||
logger.set_phase_attributes(attributes)
|
||||
|
||||
|
||||
def build_otel_v2_logger(
|
||||
config: OpenTelemetryV2Config,
|
||||
callback_name: str | None = None,
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ class GenAIMapper:
|
|||
return {}
|
||||
|
||||
def _llm_call(self, data: LLMCallSpanData) -> AttributeMap:
|
||||
attrs: Final = collect(self._LLM_CALL_ATTRS, data)
|
||||
attrs: Final = {**collect(self._LLM_CALL_ATTRS, data), **data.routing_attributes}
|
||||
if data.tools:
|
||||
attrs[LiteLLM.TOOLS_DECLARED] = len(data.tools)
|
||||
attrs.update(
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.integrations.otel.model.utils import (
|
|||
as_str_mapping,
|
||||
as_str_tuple,
|
||||
)
|
||||
from litellm.integrations.otel.routing import RoutingAttributeValue, routing_decision_attributes
|
||||
|
||||
# ``RequestIdentity`` and the request-metadata translation now live in
|
||||
# :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep
|
||||
|
|
@ -436,6 +437,7 @@ class LLMCallSpanData:
|
|||
trace: TraceControls = field(default_factory=TraceControls)
|
||||
session_id: str | None = None
|
||||
embedding_output: EmbeddingOutput | None = None
|
||||
routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
|
||||
|
||||
@classmethod
|
||||
def from_standard_logging_payload(
|
||||
|
|
@ -453,6 +455,8 @@ class LLMCallSpanData:
|
|||
# model split, the response model, api base, and identity all come from
|
||||
# here rather than being re-derived from the raw payload dicts.
|
||||
context: Final = RequestContext.from_standard_logging_payload(payload)
|
||||
metadata: Final = payload.get("metadata")
|
||||
decision: Final = metadata["routing_decision"] if metadata and "routing_decision" in metadata else None
|
||||
# Normalize ``response`` to a dict once so the content/id reads below are a
|
||||
# plain ``.get`` — no repeated ``isinstance`` guards.
|
||||
raw_response: Final = payload.get("response")
|
||||
|
|
@ -496,6 +500,7 @@ class LLMCallSpanData:
|
|||
trace=trace or TraceControls(),
|
||||
session_id=session_id or None,
|
||||
embedding_output=embedding_output if capture_content else None,
|
||||
routing_attributes=routing_decision_attributes(decision),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
57
litellm/integrations/otel/routing.py
Normal file
57
litellm/integrations/otel/routing.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
RoutingAttributeValue: TypeAlias = str | int | float | bool
|
||||
|
||||
_ROUTING_SCALAR_FIELDS: Final = frozenset(
|
||||
{
|
||||
"router_model_name",
|
||||
"router_type",
|
||||
"router_config_id",
|
||||
"router_config_updated_at",
|
||||
"router_config_fingerprint",
|
||||
"routed_model",
|
||||
"cause",
|
||||
"tier",
|
||||
"tier_label",
|
||||
"request_type",
|
||||
"score",
|
||||
"classifier_model",
|
||||
"classifier_cost",
|
||||
"classifier_failure_reason",
|
||||
"classifier_error_type",
|
||||
"classifier_confidence",
|
||||
"classifier_primary_rule",
|
||||
"classifier_capability_boundary",
|
||||
"classifier_p_solve",
|
||||
"classifier_calibrated_p_solve",
|
||||
"classifier_calibration_version",
|
||||
"classifier_efficient_p_solve",
|
||||
"classifier_capable_p_solve",
|
||||
"classifier_calibrated_efficient_p_solve",
|
||||
"classifier_calibrated_capable_p_solve",
|
||||
"classifier_max_quality_gap",
|
||||
"classifier_prompt_version",
|
||||
"classifier_threshold",
|
||||
"escalated",
|
||||
"context_escalated",
|
||||
"context_escalation_original_tier",
|
||||
"reasoning_override_min_score",
|
||||
"conversation_continuing",
|
||||
"savings_baseline_model",
|
||||
"savings_baseline_deployment_id",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def routing_decision_attributes(decision: Mapping[str, object] | None) -> Mapping[str, RoutingAttributeValue]:
|
||||
if not isinstance(decision, Mapping):
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType(
|
||||
{
|
||||
f"litellm.routing.{key}": value
|
||||
for key in _ROUTING_SCALAR_FIELDS
|
||||
if isinstance(value := decision.get(key), (str, int, float, bool))
|
||||
}
|
||||
)
|
||||
|
|
@ -7,7 +7,9 @@ V2 is not the active logger — so a call site can wrap a request phase or seed
|
|||
identity unconditionally.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Generator, Mapping
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
|
|
@ -16,11 +18,18 @@ if TYPE_CHECKING:
|
|||
from opentelemetry.trace import Span
|
||||
|
||||
PhaseEventAttributes: TypeAlias = Mapping[str, str | int]
|
||||
PhaseAttributes: TypeAlias = Mapping[str, str | int | float | bool]
|
||||
|
||||
|
||||
@cache
|
||||
def _otel_runtime() -> (
|
||||
"tuple[Callable[[str], AbstractContextManager[Span | None]], Callable[..., None], Callable[[str, PhaseEventAttributes | None], None]] | None"
|
||||
tuple[
|
||||
Callable[[str], AbstractContextManager[Span | None]],
|
||||
Callable[..., None],
|
||||
Callable[[str, PhaseEventAttributes | None], None],
|
||||
Callable[[PhaseAttributes], None],
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Resolve the SDK-backed hooks once and cache the outcome, absence included.
|
||||
|
||||
|
|
@ -32,11 +41,11 @@ def _otel_runtime() -> (
|
|||
from litellm.integrations.otel import logger
|
||||
except Exception:
|
||||
return None
|
||||
return (logger.phase_span, logger.seed_request_identity, logger.phase_event)
|
||||
return (logger.phase_span, logger.seed_request_identity, logger.phase_event, logger.phase_attributes)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def phase_span(name: str) -> "Iterator[Span | None]":
|
||||
def phase_span(name: str) -> Generator[Span | None]:
|
||||
"""Run a request phase inside a live active span so its DB/service calls nest.
|
||||
|
||||
Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not
|
||||
|
|
@ -64,3 +73,10 @@ def seed_request_identity(user_api_key_dict: object, model: object = None) -> No
|
|||
if runtime is None:
|
||||
return
|
||||
runtime[1](user_api_key_dict, model=model)
|
||||
|
||||
|
||||
def phase_attributes(attributes: PhaseAttributes) -> None:
|
||||
runtime: Final = _otel_runtime()
|
||||
if runtime is None:
|
||||
return
|
||||
runtime[3](attributes)
|
||||
|
|
|
|||
|
|
@ -73,7 +73,8 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.integrations.custom_guardrail import is_guardrail_intervention
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.runtime import phase_event, phase_span
|
||||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
_get_parent_otel_span_from_kwargs,
|
||||
|
|
@ -162,6 +163,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
AUTO_ROUTER_MODEL_PREFIX,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
claimed_capabilities,
|
||||
|
|
@ -8724,6 +8726,19 @@ class Router:
|
|||
breadcrumbs: Final = (*kept_breadcrumbs, attempt_record)
|
||||
earlier: Final = request_metadata.get("request_retry_count")
|
||||
request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1
|
||||
decision: Final = request_metadata.get("routing_decision")
|
||||
router_name: Final = decision.get("router_model_name") if isinstance(decision, Mapping) else None
|
||||
if isinstance(router_name, str):
|
||||
phase_event(
|
||||
"litellm.routing.retry",
|
||||
{
|
||||
"litellm.routing.router_model_name": router_name,
|
||||
"litellm.retry.count": request_retry_count,
|
||||
"error.type": type(e).__name__,
|
||||
**({"litellm.deployment.model_group": model_group} if isinstance(model_group, str) else {}),
|
||||
**({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}),
|
||||
},
|
||||
)
|
||||
kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict
|
||||
kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap
|
||||
return kwargs
|
||||
|
|
@ -9572,9 +9587,29 @@ class Router:
|
|||
)
|
||||
registry[deployment.model_name] = [
|
||||
*registry.get(deployment.model_name, []),
|
||||
TaggedPreRoutingStrategy(tags=tags, strategy=strategy),
|
||||
TaggedPreRoutingStrategy(
|
||||
tags=tags,
|
||||
strategy=strategy,
|
||||
definition_fingerprint=self._routing_definition_fingerprint(deployment),
|
||||
deployment=deployment,
|
||||
),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _routing_definition_fingerprint(deployment: Deployment) -> str | None:
|
||||
"""Identify configured routing fields, excluding external files and live strategy state."""
|
||||
try:
|
||||
definition: Final = json.dumps(
|
||||
deployment.litellm_params.model_dump(
|
||||
include=STRATEGY_ROUTER_PARAM_FIELDS | {"model", "tags"}, exclude_none=True, warnings=False
|
||||
),
|
||||
sort_keys=True,
|
||||
default=Router._json_default_stable_id,
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return Router.generate_model_id(deployment.model_name, {"routing_definition": definition})
|
||||
|
||||
@staticmethod
|
||||
def _unregister_pre_routing_strategy(
|
||||
registry: dict[str, list[TaggedPreRoutingStrategy[_PreRoutingStrategyT]]],
|
||||
|
|
@ -9653,7 +9688,12 @@ class Router:
|
|||
if adaptive_router is not None:
|
||||
self.adaptive_routers[model_name] = [
|
||||
*self.adaptive_routers.get(model_name, []),
|
||||
TaggedPreRoutingStrategy(tags=tagged.tags, strategy=adaptive_router),
|
||||
TaggedPreRoutingStrategy(
|
||||
tags=tagged.tags,
|
||||
strategy=adaptive_router,
|
||||
definition_fingerprint=tagged.definition_fingerprint,
|
||||
deployment=tagged.deployment,
|
||||
),
|
||||
]
|
||||
|
||||
self._sync_adaptive_router_hooks()
|
||||
|
|
@ -14224,6 +14264,7 @@ class Router:
|
|||
spend metadata is stamped before routing and the response carries the tier group the
|
||||
strategy picked.
|
||||
"""
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
requested_registered_model_name: Final = self._get_model_from_alias(model=model) or model
|
||||
registered_model_name: Final = await self._resolve_claude_code_session_router(
|
||||
model=model,
|
||||
|
|
@ -14247,7 +14288,6 @@ class Router:
|
|||
)
|
||||
if selected_strategy is None:
|
||||
await arm_compaction(request_kwargs, None)
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
self._stamp_or_clear_metadata_key(
|
||||
request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None
|
||||
)
|
||||
|
|
@ -14259,6 +14299,26 @@ class Router:
|
|||
from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
|
||||
marker: Final = self._selected_strategy_marker_deployment(
|
||||
model=registered_model_name, strategy_tags=selected_strategy.tags, request_kwargs=request_kwargs
|
||||
)
|
||||
definition: Final = selected_strategy.deployment
|
||||
config_id: Final = definition.model_info.id if definition else None
|
||||
updated_at: Final = definition.model_info.updated_at if definition else None
|
||||
router_kind: Final = classify_strategy_router_model(definition.litellm_params.model) if definition else None
|
||||
provenance: Final[StandardLoggingRoutingDecision] = {
|
||||
"router_model_name": definition.model_name if definition else registered_model_name,
|
||||
**({"router_type": router_kind} if router_kind is not None else {}),
|
||||
**({"router_config_id": config_id} if config_id is not None else {}),
|
||||
**({"router_config_updated_at": updated_at.isoformat()} if updated_at is not None else {}),
|
||||
**(
|
||||
{"router_config_fingerprint": selected_strategy.definition_fingerprint}
|
||||
if selected_strategy.definition_fingerprint is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
phase_attributes(routing_decision_attributes(provenance))
|
||||
|
||||
reject_recursive_compactor(registered_model_name)
|
||||
await arm_compaction(
|
||||
request_kwargs,
|
||||
|
|
@ -14282,11 +14342,7 @@ class Router:
|
|||
)
|
||||
|
||||
await authorize_member_auto_router_inference(
|
||||
deployment=self._selected_strategy_marker_deployment(
|
||||
model=registered_model_name,
|
||||
strategy_tags=selected_strategy.tags,
|
||||
request_kwargs=request_kwargs,
|
||||
),
|
||||
deployment=marker,
|
||||
request_kwargs=request_kwargs,
|
||||
llm_router=self,
|
||||
)
|
||||
|
|
@ -14324,13 +14380,18 @@ class Router:
|
|||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
)
|
||||
traced_routed: Final = (
|
||||
routed.model_copy(update={"routing_decision": {**provenance, **routed.routing_decision}})
|
||||
if routed is not None and routed.routing_decision is not None
|
||||
else routed
|
||||
)
|
||||
# Routing-only compression must not leak into the response: the model call and
|
||||
# deployment-context filtering key off this field. Compared by value, since
|
||||
# pydantic rebuilds the list rather than keeping the object passed in.
|
||||
pre_routing_hook_response: Final = (
|
||||
routed.model_copy(update={"messages": messages})
|
||||
if routed is not None and routing_messages is not None and routed.messages == routing_messages
|
||||
else routed
|
||||
traced_routed.model_copy(update={"messages": messages})
|
||||
if traced_routed is not None and routing_messages is not None and traced_routed.messages == routing_messages
|
||||
else traced_routed
|
||||
)
|
||||
self._record_routing_decision(
|
||||
request_kwargs=request_kwargs,
|
||||
|
|
@ -14481,6 +14542,7 @@ class Router:
|
|||
"""
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
|
||||
phase_attributes(routing_decision_attributes(routing_decision))
|
||||
baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None
|
||||
baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None
|
||||
router_name: Final = routing_decision.get("router_model_name") if routing_decision else None
|
||||
|
|
|
|||
|
|
@ -4,12 +4,13 @@ Auto-Routing Strategy that works with a Semantic Router Config
|
|||
|
||||
import asyncio
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Optional
|
||||
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
|
||||
from litellm.exceptions import Timeout as LiteLLMTimeout
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
effective_turn_off_message_logging,
|
||||
|
|
@ -17,7 +18,12 @@ from litellm.litellm_core_utils.internal_call_metadata import (
|
|||
parent_session_kwargs,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN
|
||||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
ClassifierFailureReason,
|
||||
RoutingDecisionCause,
|
||||
StandardLoggingRoutingDecision,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from semantic_router.routers import SemanticRouter
|
||||
|
|
@ -41,6 +47,12 @@ class _CallerMetadata(LiteLLMBaseModel):
|
|||
litellm_metadata: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class _SemanticMatchOutcome(NamedTuple):
|
||||
route_name: str | None = None
|
||||
failure_reason: ClassifierFailureReason | None = None
|
||||
error_type: str | None = None
|
||||
|
||||
|
||||
class AutoRouter(CustomLogger):
|
||||
DEFAULT_AUTO_SYNC_VALUE = "local"
|
||||
|
||||
|
|
@ -72,6 +84,7 @@ class AutoRouter(CustomLogger):
|
|||
|
||||
from litellm.router_strategy.auto_router.litellm_encoder import LiteLLMRouterEncoder
|
||||
|
||||
self.model_name: Final = model_name
|
||||
self.auto_router_config_path: str | None = auto_router_config_path
|
||||
self.auto_router_config: str | None = auto_router_config
|
||||
self.auto_sync_value = self.DEFAULT_AUTO_SYNC_VALUE
|
||||
|
|
@ -205,20 +218,38 @@ class AutoRouter(CustomLogger):
|
|||
if resolved_messages is None:
|
||||
return None
|
||||
|
||||
routelayer = await self._ensure_routelayer()
|
||||
routelayer: Final = await self._ensure_routelayer()
|
||||
|
||||
message_content: Final = self._extract_text_from_messages(resolved_messages)
|
||||
route_name: Final = await self._matched_route_name(routelayer, message_content, request_kwargs)
|
||||
outcome: Final = await self._match_route(routelayer, message_content, request_kwargs)
|
||||
routed_model: Final = outcome.route_name or self.default_model
|
||||
cause: Final[RoutingDecisionCause] = (
|
||||
"semantic_error"
|
||||
if outcome.failure_reason is not None
|
||||
else "semantic_match"
|
||||
if outcome.route_name
|
||||
else "semantic_no_match"
|
||||
)
|
||||
decision: Final[StandardLoggingRoutingDecision] = {
|
||||
"router_model_name": self.model_name,
|
||||
"router_type": "semantic",
|
||||
"routed_model": routed_model,
|
||||
"cause": cause,
|
||||
"classifier_model": self.embedding_model,
|
||||
**({"classifier_failure_reason": outcome.failure_reason} if outcome.failure_reason is not None else {}),
|
||||
**({"classifier_error_type": outcome.error_type} if outcome.error_type is not None else {}),
|
||||
}
|
||||
|
||||
return PreRoutingHookResponse(
|
||||
model=route_name or self.default_model,
|
||||
model=routed_model,
|
||||
messages=messages,
|
||||
routing_decision=decision,
|
||||
)
|
||||
|
||||
async def _matched_route_name(
|
||||
async def _match_route(
|
||||
self, routelayer: "SemanticRouter", text: str, request_kwargs: Mapping[str, object]
|
||||
) -> str | None:
|
||||
"""Name of the route `text` matches, or None when nothing matched or the match failed.
|
||||
) -> _SemanticMatchOutcome:
|
||||
"""Matched route or the reason a default model will serve the request.
|
||||
|
||||
`text` is embedded here rather than by `routelayer(text=...)` so the caller's metadata reaches
|
||||
`aembedding()` and the embedding's spend lands on the key/team that sent the request;
|
||||
|
|
@ -247,10 +278,16 @@ class AutoRouter(CustomLogger):
|
|||
verbose_router_logger.warning(
|
||||
"AutoRouter: semantic routing failed (%s), falling back to default model %s", e, self.default_model
|
||||
)
|
||||
return None
|
||||
failure: Final = e.__cause__ if isinstance(e, ValueError) and e.__cause__ is not None else e
|
||||
return _SemanticMatchOutcome(
|
||||
failure_reason="timeout"
|
||||
if isinstance(failure, (TimeoutError, asyncio.TimeoutError, LiteLLMTimeout))
|
||||
else "classifier_error",
|
||||
error_type=type(failure).__name__,
|
||||
)
|
||||
verbose_router_logger.debug("route_choice: %s", route_choice)
|
||||
if isinstance(route_choice, RouteChoice):
|
||||
return route_choice.name
|
||||
return _SemanticMatchOutcome(route_name=route_choice.name)
|
||||
if isinstance(route_choice, list) and route_choice:
|
||||
return route_choice[0].name
|
||||
return None
|
||||
return _SemanticMatchOutcome(route_name=route_choice[0].name)
|
||||
return _SemanticMatchOutcome()
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from litellm.constants import (
|
|||
SESSION_ID_GENERATED_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.runtime import phase_event
|
||||
from litellm.litellm_core_utils.classifier_logging import masked_originating_request
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
_get_parent_otel_span_from_kwargs,
|
||||
|
|
@ -77,6 +78,7 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
ClassifierFailureReason,
|
||||
ModelResponse,
|
||||
RoutingDecisionCause,
|
||||
StandardLoggingHeuristicV2Forecast,
|
||||
|
|
@ -1085,10 +1087,21 @@ class ClassificationOutcome(NamedTuple):
|
|||
llm_v2_forecast: LLMV2Decision | None = None
|
||||
jev_verdict: JevVerdict | None = None
|
||||
heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None
|
||||
classifier_failure_reason: ClassifierFailureReason | None = None
|
||||
classifier_error_type: str | None = None
|
||||
|
||||
|
||||
def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome:
|
||||
return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal))
|
||||
def _with_classifier_failure(
|
||||
outcome: ClassificationOutcome,
|
||||
failure_reason: ClassifierFailureReason,
|
||||
error_type: str | None,
|
||||
signal: str | None,
|
||||
) -> ClassificationOutcome:
|
||||
return outcome._replace(
|
||||
signals=(*outcome.signals, signal) if signal is not None else outcome.signals,
|
||||
classifier_failure_reason=failure_reason,
|
||||
classifier_error_type=error_type,
|
||||
)
|
||||
|
||||
|
||||
def _with_llm_v2_forecast(
|
||||
|
|
@ -1221,6 +1234,12 @@ def _is_classifier_timeout(exc: BaseException) -> bool:
|
|||
return isinstance(exc, LiteLLMTimeout)
|
||||
|
||||
|
||||
def _classifier_failure_reason(exc: Exception) -> ClassifierFailureReason:
|
||||
if _is_classifier_timeout(exc):
|
||||
return "timeout"
|
||||
return "invalid_response" if isinstance(exc, ValidationError) else "classifier_error"
|
||||
|
||||
|
||||
def _allowed(models: tuple[str, ...], fit_filter: frozenset[str] | None) -> tuple[str, ...]:
|
||||
return models if fit_filter is None else tuple(model for model in models if model in fit_filter)
|
||||
|
||||
|
|
@ -1835,6 +1854,8 @@ class ComplexityRouter(CustomLogger):
|
|||
escalated: bool = False,
|
||||
classifier_model: str | None = None,
|
||||
classifier_cost: float | None = None,
|
||||
classifier_failure_reason: ClassifierFailureReason | None = None,
|
||||
classifier_error_type: str | None = None,
|
||||
conversation_continuing: bool = True,
|
||||
tier_litellm_params: Mapping[str, object] | None = None,
|
||||
context_escalation_original_tier: ComplexityTier | str | None = None,
|
||||
|
|
@ -1853,6 +1874,12 @@ class ComplexityRouter(CustomLogger):
|
|||
routed_model=routed_model,
|
||||
cause=cause,
|
||||
conversation_continuing=conversation_continuing,
|
||||
**(
|
||||
{"classifier_failure_reason": classifier_failure_reason}
|
||||
if classifier_failure_reason is not None
|
||||
else {}
|
||||
),
|
||||
**({"classifier_error_type": classifier_error_type} if classifier_error_type is not None else {}),
|
||||
)
|
||||
if (baseline := self.savings_baseline) is not None:
|
||||
decision["savings_baseline_model"] = baseline.model
|
||||
|
|
@ -2094,7 +2121,9 @@ class ComplexityRouter(CustomLogger):
|
|||
permit: Final = breaker.acquire_permit() if breaker is not None else None
|
||||
if breaker is not None and permit is None:
|
||||
return self._capability_classifier_failure_outcome(
|
||||
"capability classifier circuit is open", signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL
|
||||
"capability classifier circuit is open",
|
||||
failure_reason="circuit_open",
|
||||
signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL,
|
||||
)
|
||||
try:
|
||||
tier, classifier_cost, forecast = await self._classify_with_capability_llm(prompt, request_kwargs, messages)
|
||||
|
|
@ -2118,9 +2147,20 @@ class ComplexityRouter(CustomLogger):
|
|||
except Exception as e: # noqa: BLE001 -- every unavailable or invalid judge verdict must fail closed
|
||||
if breaker is not None and permit is not None:
|
||||
breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
|
||||
return self._capability_classifier_failure_outcome(f"capability classifier failed ({type(e).__name__})")
|
||||
return self._capability_classifier_failure_outcome(
|
||||
f"capability classifier failed ({type(e).__name__})",
|
||||
failure_reason=_classifier_failure_reason(e),
|
||||
error_type=type(e).__name__,
|
||||
)
|
||||
|
||||
def _capability_classifier_failure_outcome(self, reason: str, signal: str | None = None) -> ClassificationOutcome:
|
||||
def _capability_classifier_failure_outcome(
|
||||
self,
|
||||
reason: str,
|
||||
*,
|
||||
failure_reason: ClassifierFailureReason,
|
||||
error_type: str | None = None,
|
||||
signal: str | None = None,
|
||||
) -> ClassificationOutcome:
|
||||
"""Fail closed to the configured capable tier without consulting another taxonomy."""
|
||||
capability: Final = self.config.capability_classifier_config
|
||||
if capability is None:
|
||||
|
|
@ -2141,6 +2181,8 @@ class ComplexityRouter(CustomLogger):
|
|||
score=None,
|
||||
signals=signals,
|
||||
cause="capability_classifier_fallback",
|
||||
classifier_failure_reason=failure_reason,
|
||||
classifier_error_type=error_type,
|
||||
)
|
||||
|
||||
async def _llm_classifier_outcome(
|
||||
|
|
@ -2164,6 +2206,7 @@ class ComplexityRouter(CustomLogger):
|
|||
prompt,
|
||||
system_prompt,
|
||||
scored,
|
||||
failure_reason="circuit_open",
|
||||
signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL,
|
||||
)
|
||||
try:
|
||||
|
|
@ -2193,7 +2236,12 @@ class ComplexityRouter(CustomLogger):
|
|||
if breaker is not None and permit is not None:
|
||||
breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
|
||||
return self._classifier_failure_outcome(
|
||||
f"LLM classifier failed ({type(e).__name__})", prompt, system_prompt, scored
|
||||
f"LLM classifier failed ({type(e).__name__})",
|
||||
prompt,
|
||||
system_prompt,
|
||||
scored,
|
||||
failure_reason=_classifier_failure_reason(e),
|
||||
error_type=type(e).__name__,
|
||||
)
|
||||
|
||||
async def _jev_classifier_outcome(
|
||||
|
|
@ -2206,11 +2254,16 @@ class ComplexityRouter(CustomLogger):
|
|||
config: Final = self.config.opensource_classifier_config
|
||||
client: Final = self._jev_client
|
||||
if config is None or client is None:
|
||||
return self._classifier_failure_outcome("jev classifier is not configured", prompt, system_prompt)
|
||||
return self._classifier_failure_outcome(
|
||||
"jev classifier is not configured", prompt, system_prompt, failure_reason="not_configured"
|
||||
)
|
||||
marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or EMPTY_MAPPING)
|
||||
if _encrypted_classifier_task(request_kwargs, marker_pairs) is not None:
|
||||
return self._classifier_failure_outcome(
|
||||
"jev classifier does not support encrypted agent tasks", prompt, system_prompt
|
||||
"jev classifier does not support encrypted agent tasks",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason="unsupported_input",
|
||||
)
|
||||
breaker: Final = self._classifier_circuit_breaker
|
||||
permit: Final = breaker.acquire_permit() if breaker is not None else None
|
||||
|
|
@ -2219,6 +2272,7 @@ class ComplexityRouter(CustomLogger):
|
|||
"jev classifier circuit is open",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason="circuit_open",
|
||||
signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL,
|
||||
)
|
||||
criteria: Final[Mapping[str, str]] = (
|
||||
|
|
@ -2288,7 +2342,11 @@ class ComplexityRouter(CustomLogger):
|
|||
if breaker is not None and permit is not None:
|
||||
breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
|
||||
return self._classifier_failure_outcome(
|
||||
f"jev classifier failed ({type(e).__name__})", prompt, system_prompt
|
||||
f"jev classifier failed ({type(e).__name__})",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason=_classifier_failure_reason(e),
|
||||
error_type=type(e).__name__,
|
||||
)
|
||||
|
||||
def _classifier_failure_outcome(
|
||||
|
|
@ -2297,6 +2355,9 @@ class ComplexityRouter(CustomLogger):
|
|||
prompt: str,
|
||||
system_prompt: str | None,
|
||||
scored: ClassificationOutcome | None = None,
|
||||
*,
|
||||
failure_reason: ClassifierFailureReason,
|
||||
error_type: str | None = None,
|
||||
signal: str | None = None,
|
||||
) -> ClassificationOutcome:
|
||||
"""The outcome when the LLM classifier or classifier plugin produced no usable tier:
|
||||
|
|
@ -2307,35 +2368,44 @@ class ComplexityRouter(CustomLogger):
|
|||
v2: Final = self.config.llm_v2_config
|
||||
if v2 is not None:
|
||||
verbose_router_logger.warning("ComplexityRouter: %s, routing to llm_v2 capable tier", reason)
|
||||
return _with_signal(
|
||||
return _with_classifier_failure(
|
||||
ClassificationOutcome(
|
||||
tier=ComplexityTier(v2.capable_tier),
|
||||
score=None,
|
||||
signals=("llm-v2:fallback-capable",),
|
||||
cause="llm_v2_fallback",
|
||||
),
|
||||
failure_reason,
|
||||
error_type,
|
||||
signal,
|
||||
)
|
||||
fallback_tier: Final = self.config.fallback_tier
|
||||
if fallback_tier is not None:
|
||||
verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier)
|
||||
return _with_signal(
|
||||
return _with_classifier_failure(
|
||||
ClassificationOutcome(
|
||||
tier=fallback_tier,
|
||||
score=None,
|
||||
signals=(f"classifier-fallback:{fallback_tier}",),
|
||||
cause="classifier_fallback",
|
||||
),
|
||||
failure_reason,
|
||||
error_type,
|
||||
signal,
|
||||
)
|
||||
verbose_router_logger.warning(
|
||||
"ComplexityRouter: %s, falling back to %s", reason, self.config.classifier_fallback
|
||||
)
|
||||
if self.config.classifier_fallback == "default_model":
|
||||
return _with_signal(self._default_model_fallback_outcome(), signal)
|
||||
return _with_classifier_failure(self._default_model_fallback_outcome(), failure_reason, error_type, signal)
|
||||
if scored is not None:
|
||||
return _with_signal(scored, signal)
|
||||
return _with_signal(self._classify_locally(prompt, system_prompt), signal)
|
||||
return _with_classifier_failure(scored, failure_reason, error_type, signal)
|
||||
return _with_classifier_failure(
|
||||
self._classify_locally(prompt, system_prompt),
|
||||
failure_reason,
|
||||
error_type,
|
||||
signal,
|
||||
)
|
||||
|
||||
async def _classify_with_plugin(
|
||||
self,
|
||||
|
|
@ -2349,7 +2419,9 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
plugin: Final = self.config.classifier_plugin
|
||||
if plugin is None:
|
||||
return self._classifier_failure_outcome("classifier_plugin is not set", prompt, system_prompt)
|
||||
return self._classifier_failure_outcome(
|
||||
"classifier_plugin is not set", prompt, system_prompt, failure_reason="not_configured"
|
||||
)
|
||||
kwargs: Final = request_kwargs if request_kwargs is not None else EMPTY_MAPPING
|
||||
pools: Final = self._tier_pools()
|
||||
try:
|
||||
|
|
@ -2365,29 +2437,48 @@ class ComplexityRouter(CustomLogger):
|
|||
verdict: Final = await asyncio.wait_for(
|
||||
plugin.classify(context), timeout=self.config.classifier_plugin_timeout_ms / 1000
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
except asyncio.TimeoutError as e:
|
||||
return self._classifier_failure_outcome(
|
||||
f"classifier plugin timed out after {self.config.classifier_plugin_timeout_ms}ms", prompt, system_prompt
|
||||
f"classifier plugin timed out after {self.config.classifier_plugin_timeout_ms}ms",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason="timeout",
|
||||
error_type=type(e).__name__,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 -- an operator hook can fail in arbitrary ways (network, bug); any failure must fall back rather than fail the request
|
||||
return self._classifier_failure_outcome(f"classifier plugin failed ({e})", prompt, system_prompt)
|
||||
return self._classifier_failure_outcome(
|
||||
f"classifier plugin failed ({e})",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason=_classifier_failure_reason(e),
|
||||
error_type=type(e).__name__,
|
||||
)
|
||||
if verdict is None:
|
||||
return self._classifier_failure_outcome("classifier plugin declined to classify", prompt, system_prompt)
|
||||
return self._classifier_failure_outcome(
|
||||
"classifier plugin declined to classify", prompt, system_prompt, failure_reason="declined"
|
||||
)
|
||||
if not isinstance(verdict, str):
|
||||
return self._classifier_failure_outcome(
|
||||
f"classifier plugin returned a non-string verdict of type {type(verdict).__name__}",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason="invalid_response",
|
||||
)
|
||||
tier: Final = self.config.resolve_classified_tier(verdict)
|
||||
if tier is None:
|
||||
return self._classifier_failure_outcome(
|
||||
f"classifier plugin returned unknown tier {verdict!r}", prompt, system_prompt
|
||||
f"classifier plugin returned unknown tier {verdict!r}",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason="invalid_response",
|
||||
)
|
||||
tier_key: Final = _tier_name(tier)
|
||||
if not pools.get(tier_key):
|
||||
return self._classifier_failure_outcome(
|
||||
f"classifier plugin returned tier {tier_key!r}, which has no models configured", prompt, system_prompt
|
||||
f"classifier plugin returned tier {tier_key!r}, which has no models configured",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason="invalid_response",
|
||||
)
|
||||
return ClassificationOutcome(
|
||||
tier=tier,
|
||||
|
|
@ -2651,9 +2742,13 @@ class ComplexityRouter(CustomLogger):
|
|||
verdict: Final = LLMV2Verdict.model_validate_json(extract_classifier_json(content))
|
||||
except ValidationError as error:
|
||||
_log_rejected_classifier_verdict(error, content, request_kwargs)
|
||||
return self._classifier_failure_outcome("Invalid LLM V2 forecast", prompt, system_prompt)._replace(
|
||||
classifier_cost=classifier_cost
|
||||
)
|
||||
return self._classifier_failure_outcome(
|
||||
"Invalid LLM V2 forecast",
|
||||
prompt,
|
||||
system_prompt,
|
||||
failure_reason="invalid_response",
|
||||
error_type=type(error).__name__,
|
||||
)._replace(classifier_cost=classifier_cost)
|
||||
decision: Final = v2.classify(verdict)
|
||||
return ClassificationOutcome(
|
||||
tier=ComplexityTier(v2.efficient_tier if decision.use_efficient else v2.capable_tier),
|
||||
|
|
@ -4094,6 +4189,10 @@ class ComplexityRouter(CustomLogger):
|
|||
try:
|
||||
semantic_tier: Final = await self._semantic_tier_override(user_message, request_kwargs)
|
||||
except Exception as e: # noqa: BLE001 -- embedding call can fail many ways (timeout, provider/network/parse error); any failure must fall back to scoring, never fail the request
|
||||
phase_event(
|
||||
"litellm.router.semantic_keyword_fallback",
|
||||
{"litellm.router.failure_reason": _classifier_failure_reason(e), "error.type": type(e).__name__},
|
||||
)
|
||||
verbose_router_logger.warning(
|
||||
"ComplexityRouter: semantic keyword matching failed (%s), falling back to complexity scoring", e
|
||||
)
|
||||
|
|
@ -4728,6 +4827,8 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing=conversation_continuing,
|
||||
cause=outcome.cause,
|
||||
signals=outcome.signals,
|
||||
classifier_failure_reason=outcome.classifier_failure_reason,
|
||||
classifier_error_type=outcome.classifier_error_type,
|
||||
escalation_keyword=escalation_keyword,
|
||||
escalated=False,
|
||||
tier_litellm_params=fallback_tier_params,
|
||||
|
|
@ -4840,6 +4941,8 @@ class ComplexityRouter(CustomLogger):
|
|||
escalated=escalated,
|
||||
classifier_model=classifier_model,
|
||||
classifier_cost=outcome.classifier_cost,
|
||||
classifier_failure_reason=outcome.classifier_failure_reason,
|
||||
classifier_error_type=outcome.classifier_error_type,
|
||||
tier_litellm_params=tier_litellm_params,
|
||||
context_escalation_original_tier=context_original_tier,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1214,6 +1214,8 @@ class TaggedPreRoutingStrategy(Generic[_PreRoutingStrategyT_co]):
|
|||
|
||||
tags: tuple[str, ...]
|
||||
strategy: _PreRoutingStrategyT_co
|
||||
definition_fingerprint: str | None = None
|
||||
deployment: Deployment | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -3061,6 +3061,14 @@ RoutingDecisionCause = Literal[
|
|||
"keyword",
|
||||
"quality_tier",
|
||||
"bandit",
|
||||
"semantic_match",
|
||||
"semantic_no_match",
|
||||
"semantic_error",
|
||||
]
|
||||
|
||||
|
||||
ClassifierFailureReason = Literal[
|
||||
"timeout", "circuit_open", "not_configured", "unsupported_input", "invalid_response", "declined", "classifier_error"
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -3093,7 +3101,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
|
|||
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""
|
||||
|
||||
router_model_name: str
|
||||
router_type: Literal["complexity", "adaptive", "quality"]
|
||||
router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]]
|
||||
router_config_id: ReadOnly[str]
|
||||
router_config_updated_at: ReadOnly[str]
|
||||
router_config_fingerprint: ReadOnly[str]
|
||||
routed_model: str
|
||||
cause: RoutingDecisionCause
|
||||
tier: str
|
||||
|
|
@ -3105,6 +3116,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
|
|||
escalation_keyword: str
|
||||
classifier_model: str
|
||||
classifier_cost: float
|
||||
classifier_failure_reason: ReadOnly[ClassifierFailureReason]
|
||||
classifier_error_type: ReadOnly[str]
|
||||
classifier_probabilities: ReadOnly[Mapping[str, float]]
|
||||
classifier_confidence: ReadOnly[float]
|
||||
heuristic_v2_forecast: ReadOnly[StandardLoggingHeuristicV2Forecast]
|
||||
|
|
@ -3143,6 +3156,9 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
{
|
||||
"router_model_name",
|
||||
"router_type",
|
||||
"router_config_id",
|
||||
"router_config_updated_at",
|
||||
"router_config_fingerprint",
|
||||
"routed_model",
|
||||
"cause",
|
||||
"tier",
|
||||
|
|
@ -3151,6 +3167,8 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
"score",
|
||||
"classifier_model",
|
||||
"classifier_cost",
|
||||
"classifier_failure_reason",
|
||||
"classifier_error_type",
|
||||
"classifier_probabilities",
|
||||
"classifier_confidence",
|
||||
"heuristic_v2_forecast",
|
||||
|
|
|
|||
|
|
@ -182,10 +182,63 @@ def test_async_log_success_event_emits_llm_call_span():
|
|||
assert span.attributes[GenAI.OPERATION_NAME] == "chat"
|
||||
assert span.attributes[GenAI.REQUEST_MODEL] == "gpt-4o"
|
||||
assert span.attributes[LiteLLM.CALL_ID] == "call_1"
|
||||
assert not any(key.startswith("litellm.routing.") for key in span.attributes)
|
||||
# Success leaves status UNSET (semconv default), not forced OK.
|
||||
assert span.status.status_code is StatusCode.UNSET
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", [False, True])
|
||||
@pytest.mark.parametrize("legacy_compat", [False, True])
|
||||
def test_routing_diagnostics_reach_llm_spans_without_promoting_prompt_text(fail: bool, legacy_compat: bool) -> None:
|
||||
logger, exporter = _logger(legacy_compat=legacy_compat)
|
||||
decision: Final = {
|
||||
"router_model_name": "auto-router",
|
||||
"router_type": "complexity",
|
||||
"router_config_id": "definition-1",
|
||||
"router_config_updated_at": "2026-10-06T00:00:00Z",
|
||||
"router_config_fingerprint": "definition-hash",
|
||||
"routed_model": "answer",
|
||||
"cause": "heuristic",
|
||||
"score": 0.25,
|
||||
"classifier_cost": 0.001,
|
||||
"classifier_failure_reason": "timeout",
|
||||
"classifier_error_type": "TimeoutError",
|
||||
"escalated": False,
|
||||
}
|
||||
payload: Final = _payload(
|
||||
metadata={
|
||||
"routing_decision": {
|
||||
**decision,
|
||||
"signals": ["private prompt"],
|
||||
"matched_keyword": "private prompt",
|
||||
"escalation_keyword": "private prompt",
|
||||
"classifier_crux": "private prompt",
|
||||
"tier_litellm_params": {"api_key": "private key"},
|
||||
"classifier_probabilities": {"easy": 0.25},
|
||||
"unknown_field": "private config",
|
||||
"tier_label": {"invalid": "scalar field"},
|
||||
}
|
||||
},
|
||||
status="failure" if fail else "success",
|
||||
)
|
||||
_emit_llm(logger, _kwargs(payload), fail=fail)
|
||||
(span,) = exporter.get_finished_spans()
|
||||
actual: Final = {key: value for key, value in span.attributes.items() if key.startswith("litellm.routing.")}
|
||||
assert actual == {f"litellm.routing.{key}": value for key, value in decision.items()}
|
||||
assert isinstance(actual["litellm.routing.score"], float)
|
||||
assert actual["litellm.routing.escalated"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decision", ["custom", 42, ["custom"]])
|
||||
def test_malformed_routing_metadata_preserves_the_completed_llm_span(decision: object) -> None:
|
||||
logger, exporter = _logger()
|
||||
_emit_llm(logger, _kwargs(_payload(metadata={"routing_decision": decision})))
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert span.attributes[GenAI.REQUEST_MODEL] == "gpt-4o"
|
||||
assert span.attributes[LiteLLM.CALL_ID] == "call_1"
|
||||
assert not any(key.startswith("litellm.routing.") for key in span.attributes)
|
||||
|
||||
|
||||
def test_llm_call_span_carries_the_callers_conversation_id():
|
||||
logger, exporter = _logger()
|
||||
kwargs = {**_kwargs(), "litellm_params": {"litellm_session_id": "conv-42", "metadata": {}}}
|
||||
|
|
@ -1103,6 +1156,29 @@ def test_phase_event_lands_on_root_span_even_inside_active_phase_span():
|
|||
assert by_name["auth /chat/completions"].events == ()
|
||||
|
||||
|
||||
def test_routing_attributes_stay_on_the_active_phase_while_retry_events_stay_on_root() -> None:
|
||||
logger, exporter = _logger()
|
||||
root: Final = logger.tracer.start_span("request", kind=SpanKind.SERVER)
|
||||
set_request_root_span(root)
|
||||
with trace.use_span(root, end_on_exit=True):
|
||||
logger.set_phase_attributes({"litellm.routing.cause": "outside phase"})
|
||||
with logger.start_phase_span("route auto-router"):
|
||||
logger.set_phase_attributes({"litellm.routing.router_config_id": "definition-1"})
|
||||
with logger.tracer.start_as_current_span("classifier transport"):
|
||||
logger.set_phase_attributes({"litellm.routing.cause": "heuristic"})
|
||||
logger.add_phase_event("litellm.request.retry", {"error.type": "TimeoutError"})
|
||||
logger.set_phase_attributes({"litellm.routing.cause": "after phase"})
|
||||
by_name: Final = {span.name: span for span in exporter.get_finished_spans()}
|
||||
route: Final = by_name["route auto-router"]
|
||||
assert route.attributes["litellm.routing.router_config_id"] == "definition-1"
|
||||
assert route.attributes["litellm.routing.cause"] == "heuristic"
|
||||
assert route.parent.span_id == root.get_span_context().span_id
|
||||
assert route.events == ()
|
||||
assert "litellm.routing.cause" not in by_name["request"].attributes
|
||||
assert "litellm.routing.cause" not in by_name["classifier transport"].attributes
|
||||
assert by_name["request"].events[0].attributes["error.type"] == "TimeoutError"
|
||||
|
||||
|
||||
def test_live_llm_span_anchors_to_root_with_no_active_span():
|
||||
"""Bug 2 (pass-through), live path: even with no span active at ``pre_call``,
|
||||
the anchor is a recordable parent, so the span opens live under the server root
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@ import lock. These tests pin the import to a single resolution.
|
|||
import builtins
|
||||
import importlib.abc
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from types import ModuleType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm.integrations.otel.runtime as runtime
|
||||
|
||||
|
|
@ -92,4 +97,28 @@ def test_phase_span_does_not_import_the_proxy_in_an_sdk_process(monkeypatch):
|
|||
with runtime.phase_span("route gpt-5-mini") as span:
|
||||
assert span is None
|
||||
|
||||
assert runtime.phase_attributes({"litellm.routing.score": 0.25}) is None
|
||||
assert proxy_imports == []
|
||||
|
||||
|
||||
def test_phase_attributes_no_op_when_sdk_import_is_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
runtime._otel_runtime.cache_clear()
|
||||
real_import: Final = builtins.__import__
|
||||
|
||||
def without_sdk(
|
||||
name: str,
|
||||
globals: dict[str, object] | None = None,
|
||||
locals: dict[str, object] | None = None,
|
||||
fromlist: Sequence[str] = (),
|
||||
level: int = 0,
|
||||
) -> ModuleType:
|
||||
if name == "litellm.integrations.otel" and "logger" in fromlist:
|
||||
raise ImportError("OpenTelemetry SDK is not installed")
|
||||
return real_import(name, globals, locals, fromlist, level)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", without_sdk)
|
||||
try:
|
||||
assert runtime.phase_attributes({"litellm.routing.score": 0.25}) is None
|
||||
assert runtime._otel_runtime() is None
|
||||
finally:
|
||||
runtime._otel_runtime.cache_clear()
|
||||
|
|
|
|||
|
|
@ -310,6 +310,65 @@ class TestOpenTelemetryTeamAttributesOnChildSpans(unittest.TestCase):
|
|||
)
|
||||
|
||||
|
||||
def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metadata() -> None:
|
||||
exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = TracerProvider()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
otel: Final = OpenTelemetry(tracer_provider=provider)
|
||||
decision: Final = {
|
||||
"router_model_name": "auto-router",
|
||||
"routed_model": "answer",
|
||||
"router_config_id": "definition-1",
|
||||
"cause": "heuristic",
|
||||
"classifier_failure_reason": "timeout",
|
||||
"classifier_error_type": "TimeoutError",
|
||||
"classifier_cost": 0.001,
|
||||
}
|
||||
with provider.get_tracer(__name__).start_as_current_span("inference") as span:
|
||||
otel.set_attributes(
|
||||
span,
|
||||
{
|
||||
"model": "answer",
|
||||
"standard_logging_object": {
|
||||
"id": "routing-call",
|
||||
"call_type": "acompletion",
|
||||
"metadata": {"routing_decision": decision},
|
||||
},
|
||||
},
|
||||
{"model": "answer", "choices": [], "usage": {}},
|
||||
)
|
||||
(finished,) = exporter.get_finished_spans()
|
||||
actual: Final = {
|
||||
key: value for key, value in finished.attributes.items() if key.startswith("litellm.routing.")
|
||||
}
|
||||
assert actual == {f"litellm.routing.{key}": value for key, value in decision.items()}
|
||||
assert finished.attributes["metadata.routing_decision"] == str(decision)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decision", ["custom", 42, ["custom"]])
|
||||
def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: object) -> None:
|
||||
exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = TracerProvider()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
otel: Final = OpenTelemetry(tracer_provider=provider)
|
||||
with provider.get_tracer(__name__).start_as_current_span("inference") as span:
|
||||
otel.set_attributes(
|
||||
span,
|
||||
{
|
||||
"model": "answer",
|
||||
"standard_logging_object": {
|
||||
"id": "routing-call",
|
||||
"call_type": "acompletion",
|
||||
"metadata": {"routing_decision": decision},
|
||||
},
|
||||
},
|
||||
{"model": "answer", "choices": [], "usage": {}},
|
||||
)
|
||||
(finished,) = exporter.get_finished_spans()
|
||||
assert finished.attributes["gen_ai.request.model"] == "answer"
|
||||
assert not any(key.startswith("litellm.routing.") for key in finished.attributes)
|
||||
|
||||
|
||||
class TestOpenTelemetryCostBreakdown(unittest.TestCase):
|
||||
def test_cost_breakdown_emitted_to_otel_span(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1222,15 +1222,14 @@ class TestAutoRouterBenchmarks:
|
|||
assert [group.router_name for group in response.groups] == ["tagged"]
|
||||
|
||||
def test_the_listed_kinds_match_the_router_types_traffic_can_record(self):
|
||||
"""The one reason semantic is excluded, pinned against both declarations: a kind the
|
||||
rollup can record must be listable, and a kind it cannot must not be."""
|
||||
from typing import get_args, get_type_hints
|
||||
|
||||
from litellm.router_utils.auto_router_model_naming import StrategyRouterKind
|
||||
from litellm.types.utils import StandardLoggingRoutingDecision
|
||||
|
||||
recorded = set(get_args(get_type_hints(StandardLoggingRoutingDecision)["router_type"]))
|
||||
assert set(get_args(StrategyRouterKind)) - {"semantic"} == recorded
|
||||
readonly_router_type: Final = get_type_hints(StandardLoggingRoutingDecision, include_extras=True)["router_type"]
|
||||
recorded: Final = set(get_args(get_args(readonly_router_type)[0]))
|
||||
assert set(get_args(StrategyRouterKind)) == recorded
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import pytest
|
||||
|
||||
|
||||
from litellm.exceptions import Timeout as LiteLLMTimeout
|
||||
from litellm.router_strategy.auto_router.auto_router import AutoRouter
|
||||
|
||||
pytestmark_skip_beta = pytest.mark.skip(
|
||||
|
|
@ -342,6 +343,8 @@ class FixedRouteLayer:
|
|||
self.route_choice = route_choice
|
||||
|
||||
async def acall(self, vector: Any) -> Any:
|
||||
if isinstance(self.route_choice, BaseException):
|
||||
raise self.route_choice
|
||||
return self.route_choice
|
||||
|
||||
|
||||
|
|
@ -372,11 +375,15 @@ class StubEmbeddingRouter:
|
|||
class FailingEmbeddingRouter(StubEmbeddingRouter):
|
||||
"""Router whose query embedding fails, as it does when the prompt exceeds the encoder's window."""
|
||||
|
||||
async def aembedding(self, input: List[str], model: str, **kwargs: Any) -> Any:
|
||||
raise ValueError(
|
||||
def __init__(self, error: Exception | None = None) -> None:
|
||||
super().__init__()
|
||||
self.error = error or ValueError(
|
||||
"litellm.InternalServerError: input is too large to process. increase the physical batch size"
|
||||
)
|
||||
|
||||
async def aembedding(self, input: List[str], model: str, **kwargs: Any) -> Any:
|
||||
raise self.error
|
||||
|
||||
|
||||
def _auto_router(routelayer: Any, litellm_router_instance: Any = None, **kwargs: Any) -> AutoRouter:
|
||||
auto_router: Final = AutoRouter(
|
||||
|
|
@ -395,24 +402,35 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
|
|||
"""The hook returns the alias's default model instead of failing or leaking the alias downstream."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_fall_back_to_default_model_when_the_embedding_call_fails(self):
|
||||
@pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"])
|
||||
async def test_should_fall_back_to_default_model_when_the_embedding_call_fails(self, request_model: str) -> None:
|
||||
auto_router: Final = _auto_router(FixedRouteLayer(None), litellm_router_instance=FailingEmbeddingRouter())
|
||||
|
||||
result: Final = await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router",
|
||||
model=request_model,
|
||||
request_kwargs={},
|
||||
messages=[{"role": "user", "content": "a" * 100_000}],
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.model == "fallback-model"
|
||||
assert result.routing_decision == {
|
||||
"router_model_name": "my-auto-router",
|
||||
"router_type": "semantic",
|
||||
"routed_model": "fallback-model",
|
||||
"cause": "semantic_error",
|
||||
"classifier_model": auto_router.embedding_model,
|
||||
"classifier_failure_reason": "classifier_error",
|
||||
"classifier_error_type": "ValueError",
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_fall_back_to_default_model_when_no_route_matches(self):
|
||||
@pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"])
|
||||
async def test_should_fall_back_to_default_model_when_no_route_matches(self, request_model: str) -> None:
|
||||
auto_router: Final = _auto_router(FixedRouteLayer(None))
|
||||
|
||||
result: Final = await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router",
|
||||
model=request_model,
|
||||
request_kwargs={},
|
||||
messages=[{"role": "user", "content": "nothing like any route"}],
|
||||
)
|
||||
|
|
@ -420,6 +438,13 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
|
|||
assert result is not None
|
||||
# Leaving "my-auto-router" here fails downstream with "Unmapped LLM provider".
|
||||
assert result.model == "fallback-model"
|
||||
assert result.routing_decision == {
|
||||
"router_model_name": "my-auto-router",
|
||||
"router_type": "semantic",
|
||||
"routed_model": "fallback-model",
|
||||
"cause": "semantic_no_match",
|
||||
"classifier_model": auto_router.embedding_model,
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_fall_back_to_default_model_when_the_route_layer_returns_an_empty_list(self):
|
||||
|
|
@ -452,14 +477,15 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
|
|||
assert result.model == "code-model"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_still_route_to_the_matched_route_when_one_matches(self):
|
||||
@pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"])
|
||||
async def test_should_still_route_to_the_matched_route_when_one_matches(self, request_model: str) -> None:
|
||||
from semantic_router.schema import RouteChoice
|
||||
|
||||
router: Final = StubEmbeddingRouter()
|
||||
auto_router: Final = _auto_router(FixedRouteLayer(RouteChoice(name="code-model")), litellm_router_instance=router)
|
||||
|
||||
result: Final = await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router",
|
||||
model=request_model,
|
||||
request_kwargs={},
|
||||
messages=[{"role": "user", "content": "fix this stack trace"}],
|
||||
)
|
||||
|
|
@ -467,6 +493,43 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
|
|||
assert result is not None
|
||||
assert result.model == "code-model"
|
||||
assert router.seen_text == "fix this stack trace"
|
||||
assert result.routing_decision == {
|
||||
"router_model_name": "my-auto-router",
|
||||
"router_type": "semantic",
|
||||
"routed_model": "code-model",
|
||||
"cause": "semantic_match",
|
||||
"classifier_model": auto_router.embedding_model,
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", (
|
||||
TimeoutError("private request content"),
|
||||
asyncio.TimeoutError("private request content"),
|
||||
LiteLLMTimeout(message="private request content", model="embedding", llm_provider="openai"),
|
||||
))
|
||||
@pytest.mark.parametrize("wrapped", [False, True])
|
||||
async def test_should_record_timeout_without_exception_message(self, error: Exception, wrapped: bool) -> None:
|
||||
auto_router: Final = _auto_router(
|
||||
FixedRouteLayer(None if wrapped else error),
|
||||
litellm_router_instance=FailingEmbeddingRouter(error) if wrapped else StubEmbeddingRouter(),
|
||||
)
|
||||
result: Final = await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router", request_kwargs={}, messages=[{"role": "user", "content": "route this"}]
|
||||
)
|
||||
assert result is not None and result.model == "fallback-model"
|
||||
assert result.routing_decision is not None
|
||||
assert result.routing_decision["cause"] == "semantic_error"
|
||||
assert result.routing_decision["classifier_failure_reason"] == "timeout"
|
||||
assert result.routing_decision["classifier_error_type"] == type(error).__name__
|
||||
assert "private request content" not in str(result.routing_decision)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_propagate_cancellation_during_matching(self) -> None:
|
||||
auto_router: Final = _auto_router(FixedRouteLayer(asyncio.CancelledError()))
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router", request_kwargs={}, messages=[{"role": "user", "content": "route this"}]
|
||||
)
|
||||
|
||||
|
||||
class TestAutoRouterEmbeddingInputCap:
|
||||
|
|
|
|||
|
|
@ -20,11 +20,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import ValidationError
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.utils import StandardLoggingRoutingDecision
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
CUSTOMIZATION_CAPABILITY,
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
|
|
@ -484,6 +485,10 @@ class TestComplexityRouterInit:
|
|||
assert first.cause != "jev_classifier"
|
||||
assert second.cause != "jev_classifier"
|
||||
assert client.calls == 1
|
||||
assert first.classifier_failure_reason == "timeout"
|
||||
assert first.classifier_error_type == "TimeoutError"
|
||||
assert second.classifier_failure_reason == "circuit_open"
|
||||
assert second.classifier_error_type is None
|
||||
assert _CLASSIFIER_CIRCUIT_OPEN_SIGNAL in second.signals
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -518,6 +523,8 @@ class TestComplexityRouterInit:
|
|||
outcome = await router.aclassify("Explain this")
|
||||
|
||||
assert outcome.cause != "jev_classifier"
|
||||
assert outcome.classifier_failure_reason == "classifier_error"
|
||||
assert outcome.classifier_error_type == ("RuntimeError" if isinstance(response, RuntimeError) else "ValueError")
|
||||
|
||||
|
||||
class TestTokenScoring:
|
||||
|
|
@ -3342,6 +3349,8 @@ class TestCapabilityClassifier:
|
|||
assert outcome.tier == ComplexityTier.REASONING
|
||||
assert outcome.cause == "capability_classifier_fallback"
|
||||
assert outcome.signals == ("capability-classifier-fallback",)
|
||||
assert outcome.classifier_failure_reason == "invalid_response"
|
||||
assert outcome.classifier_error_type == "ValidationError"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_classifier_call_failure_fails_closed_to_capable_model(self, mock_router_instance):
|
||||
|
|
@ -3353,6 +3362,8 @@ class TestCapabilityClassifier:
|
|||
)
|
||||
assert response.model == "capable-model"
|
||||
assert response.routing_decision["cause"] == "capability_classifier_fallback"
|
||||
assert response.routing_decision["classifier_failure_reason"] == "timeout"
|
||||
assert response.routing_decision["classifier_error_type"] == "TimeoutError"
|
||||
assert "classifier_p_solve" not in response.routing_decision
|
||||
assert "classifier_threshold" not in response.routing_decision
|
||||
|
||||
|
|
@ -4226,6 +4237,9 @@ class TestLLMClassifier:
|
|||
assert first.cause == "heuristic_scorer"
|
||||
assert second.cause == "heuristic_scorer"
|
||||
assert "classifier-circuit-open" in second.signals
|
||||
assert first.classifier_failure_reason == "timeout"
|
||||
assert second.classifier_failure_reason == "circuit_open"
|
||||
assert second.classifier_error_type is None
|
||||
mock_router_instance.acompletion.assert_awaited_once()
|
||||
|
||||
def test_classifier_circuit_allows_one_probe_and_closes_on_success(self):
|
||||
|
|
@ -8189,18 +8203,24 @@ class TestClassifierPlugin:
|
|||
router = _plugin_router(mock_router_instance, _FixedTierClassifier(None))
|
||||
outcome = await router.aclassify("what is 2+2?")
|
||||
assert outcome.cause == "heuristic_scorer"
|
||||
assert outcome.classifier_failure_reason == "declined"
|
||||
assert outcome.classifier_error_type is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_error_falls_back_to_heuristic(self, mock_router_instance):
|
||||
router = _plugin_router(mock_router_instance, _RaisingClassifier())
|
||||
outcome = await router.aclassify("what is 2+2?")
|
||||
assert outcome.cause == "heuristic_scorer"
|
||||
assert outcome.classifier_failure_reason == "classifier_error"
|
||||
assert outcome.classifier_error_type == "RuntimeError"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_timeout_falls_back_to_heuristic(self, mock_router_instance):
|
||||
router = _plugin_router(mock_router_instance, _SlowClassifier(), classifier_plugin_timeout_ms=20)
|
||||
outcome = await router.aclassify("what is 2+2?")
|
||||
assert outcome.cause == "heuristic_scorer"
|
||||
assert outcome.classifier_failure_reason == "timeout"
|
||||
assert outcome.classifier_error_type == "TimeoutError"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_non_string_verdict_falls_back_to_heuristic(self, mock_router_instance):
|
||||
|
|
@ -8208,12 +8228,16 @@ class TestClassifierPlugin:
|
|||
router = _plugin_router(mock_router_instance, _FixedTierClassifier(42))
|
||||
outcome = await router.aclassify("what is 2+2?")
|
||||
assert outcome.cause == "heuristic_scorer"
|
||||
assert outcome.classifier_failure_reason == "invalid_response"
|
||||
assert outcome.classifier_error_type is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_unknown_tier_falls_back_to_heuristic(self, mock_router_instance):
|
||||
router = _plugin_router(mock_router_instance, _FixedTierClassifier("galactic"))
|
||||
outcome = await router.aclassify("what is 2+2?")
|
||||
assert outcome.cause == "heuristic_scorer"
|
||||
assert outcome.classifier_failure_reason == "invalid_response"
|
||||
assert outcome.classifier_error_type is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_tier_without_pool_falls_back(self, mock_router_instance):
|
||||
|
|
@ -8229,6 +8253,8 @@ class TestClassifierPlugin:
|
|||
)
|
||||
outcome = await router.aclassify("what is 2+2?")
|
||||
assert outcome.cause == "heuristic_scorer"
|
||||
assert outcome.classifier_failure_reason == "invalid_response"
|
||||
assert outcome.classifier_error_type is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_failure_with_default_model_fallback(self, mock_router_instance):
|
||||
|
|
@ -9564,6 +9590,101 @@ class TestRedactedLoggingDropsPromptText:
|
|||
assert decision["cause"] == "literal_keyword_match"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected_cause",
|
||||
[
|
||||
({"classifier_type": "llm"}, "heuristic_scorer"),
|
||||
({"classifier_type": "llm", "classifier_fallback": "default_model"}, "default_model_fallback"),
|
||||
(
|
||||
{
|
||||
"classifier_type": "capability",
|
||||
"capability_classifier_config": {
|
||||
"efficient_tier": "SIMPLE", "capable_tier": "REASONING",
|
||||
"base_threshold": 0.5, "threshold_step": 0.1,
|
||||
},
|
||||
},
|
||||
"capability_classifier_fallback",
|
||||
),
|
||||
(
|
||||
{
|
||||
"classifier_type": "llm_v2",
|
||||
"llm_v2_config": {
|
||||
"efficient_profile": "Routine tasks", "capable_profile": "Complex tasks",
|
||||
"harness": "Read files and run checks", "max_quality_gap": 0.05,
|
||||
},
|
||||
},
|
||||
"llm_v2_fallback",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_classifier_failure_survives_real_router_decision_and_redaction(
|
||||
overrides: Mapping[str, object], expected_cause: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = Router(model_list=[
|
||||
{
|
||||
"model_name": "auto",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"tiers": {"SIMPLE": "cheap", "REASONING": "capable"},
|
||||
"default_model": "cheap",
|
||||
"classifier_llm_config": {"model": "classifier", "timeout_ms": 2000},
|
||||
**overrides,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "classifier",
|
||||
"litellm_params": {
|
||||
"model": "openai/test-classifier", "api_key": "test-key",
|
||||
"api_base": "https://classifier.test/v1",
|
||||
},
|
||||
},
|
||||
{"model_name": "cheap", "litellm_params": {"model": "openai/test-cheap", "api_key": "test-key"}},
|
||||
{"model_name": "capable", "litellm_params": {"model": "openai/test-capable", "api_key": "test-key"}},
|
||||
])
|
||||
with respx.mock:
|
||||
upstream: Final = respx.post("https://classifier.test/v1/chat/completions").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "classifier-result", "object": "chat.completion", "model": "test-classifier",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "PRIVATE_INVALID_REPLY"},
|
||||
"finish_reason": "stop"}],
|
||||
},
|
||||
)
|
||||
metadata: Final[dict[str, object]] = {"headers": {"x-litellm-enable-message-redaction": True}}
|
||||
request: Final[dict[str, object]] = {"metadata": metadata, "turn_off_message_logging": True}
|
||||
response: Final = await router.async_pre_routing_hook(
|
||||
model="auto", request_kwargs=request, messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
assert upstream.called
|
||||
assert response is not None and response.routing_decision is not None
|
||||
decision: Final = TypeAdapter(StandardLoggingRoutingDecision).validate_python(metadata["routing_decision"])
|
||||
assert decision["cause"] == expected_cause
|
||||
assert decision["classifier_failure_reason"] == "invalid_response"
|
||||
assert decision["classifier_error_type"] == "ValidationError"
|
||||
assert "PRIVATE_INVALID_REPLY" not in json.dumps(decision)
|
||||
assert "signals" not in decision
|
||||
if overrides["classifier_type"] == "llm":
|
||||
upstream.respond(200, json={
|
||||
"id": "classifier-recovered", "object": "chat.completion", "model": "test-classifier",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": '{"tier":"SIMPLE"}'},
|
||||
"finish_reason": "stop"}],
|
||||
})
|
||||
recovered: Final = await router.async_pre_routing_hook(
|
||||
model="auto", request_kwargs=request, messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
assert recovered is not None and recovered.routing_decision is not None
|
||||
assert recovered.routing_decision["cause"] == "llm_classifier"
|
||||
recovered_decision: Final = TypeAdapter(StandardLoggingRoutingDecision).validate_python(
|
||||
metadata["routing_decision"]
|
||||
)
|
||||
assert "classifier_failure_reason" not in recovered_decision
|
||||
assert "classifier_error_type" not in recovered_decision
|
||||
|
||||
|
||||
def test_every_routing_decision_field_is_classified():
|
||||
"""Redaction is derived from a declaration, not a list at the call site, so every
|
||||
field has to be classified as quoting the prompt or aggregating it. A field added
|
||||
|
|
@ -14148,6 +14269,8 @@ async def test_v2_chain_judge_failure_uses_selected_fallback(
|
|||
assert outcome.cause == ("heuristic_v2" if fallback == "heuristic" else "default_model_fallback")
|
||||
assert outcome.tier == (ComplexityTier.COMPLEX if fallback == "heuristic" else ComplexityTier.MEDIUM)
|
||||
assert (outcome.heuristic_v2_forecast is not None) == (fallback == "heuristic")
|
||||
assert outcome.classifier_failure_reason == "classifier_error"
|
||||
assert outcome.classifier_error_type == "RuntimeError"
|
||||
assert dependency.aresponses.await_count == int(encrypted)
|
||||
assert dependency.acompletion.await_count == int(not encrypted)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import warnings
|
|||
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final, Literal
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -31,6 +31,7 @@ from litellm.exceptions import GuardrailRaisedException, MidStreamFallbackError,
|
|||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.plumbing.context import set_request_root_span
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
|
||||
SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
|
||||
|
|
@ -68,8 +69,14 @@ from litellm.types.router import (
|
|||
ModelInfo,
|
||||
PreRoutingHookResponse,
|
||||
RetryPolicy,
|
||||
RoutingContext,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
|
||||
def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
|
||||
# initialize a real Router (env‑vars can be empty)
|
||||
|
|
@ -13213,15 +13220,22 @@ class TestTeamPublicNameReachesPreRoutingStrategies:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_public_name_shadows_a_global_model_for_that_team_only(self):
|
||||
router = self._router(
|
||||
{self.INTERNAL_NAME: self._RewriteStrategy()},
|
||||
extra_deployments=({"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}},),
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
self._team_marker(self.INTERNAL_NAME),
|
||||
{"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}},
|
||||
{"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}},
|
||||
],
|
||||
)
|
||||
|
||||
async def routed(request_kwargs: dict) -> str | None:
|
||||
response = await router.async_pre_routing_hook(
|
||||
model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages()
|
||||
)
|
||||
if response is not None:
|
||||
assert response.routing_decision is not None
|
||||
assert response.routing_decision["router_model_name"] == self.INTERNAL_NAME
|
||||
assert response.routing_decision["router_config_id"] == router.model_list[0]["model_info"]["id"]
|
||||
return response.model if response else None
|
||||
|
||||
async def selected(request_kwargs: dict) -> str:
|
||||
|
|
@ -20267,3 +20281,270 @@ async def test_non_chat_surfaces_mark_their_deployment_pick(monkeypatch: pytest.
|
|||
router.completion(model="gpt-4o", messages=[{"role": "user", "content": "hi"}])
|
||||
|
||||
assert events == [_pick("embed", "initial", 1), _pick("gpt-4o", "initial", 1)]
|
||||
|
||||
|
||||
class TestAutoRouterTraceProvenance:
|
||||
@pytest.fixture
|
||||
def _tracing(self, monkeypatch: pytest.MonkeyPatch) -> "tuple[OpenTelemetryV2, InMemorySpanExporter]":
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from litellm.integrations.otel import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.plumbing import providers
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
config: Final = OpenTelemetryV2Config(exporter="in_memory")
|
||||
exporter: Final = InMemorySpanExporter()
|
||||
logger: Final = OpenTelemetryV2(config=config, tracer_provider=providers.build_tracer_provider(config, exporter=exporter))
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger)
|
||||
return logger, exporter
|
||||
|
||||
@staticmethod
|
||||
def _marker(tag: str, target: str = "answer") -> dict[str, object]:
|
||||
return {
|
||||
"model_name": "traced-router",
|
||||
"model_info": {"id": f"definition-{tag}", "updated_at": "2026-10-01T12:00:00+00:00"},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router", "tags": [tag],
|
||||
"complexity_router_config": {
|
||||
"tiers": dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), target),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def test_definition_fingerprint_is_canonical_and_tracks_config_not_credentials(self):
|
||||
original: Final = Deployment.model_validate(self._marker("blue"))
|
||||
reordered: Final = Deployment.model_validate({
|
||||
**self._marker("blue"),
|
||||
"litellm_params": {
|
||||
"api_key": "credential-only-change", "tags": ["blue"], "model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"tiers": dict.fromkeys(("REASONING", "COMPLEX", "MEDIUM", "SIMPLE"), "answer")},
|
||||
},
|
||||
})
|
||||
fingerprint: Final = Router._routing_definition_fingerprint(original)
|
||||
assert fingerprint is not None
|
||||
assert fingerprint == Router._routing_definition_fingerprint(reordered)
|
||||
assert fingerprint != Router._routing_definition_fingerprint(Deployment.model_validate(self._marker("blue", "other")))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_classifier_and_plugin_prelude_never_inherit_parent_decisions(
|
||||
self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
logger, exporter = _tracing
|
||||
completed: Final = asyncio.Event()
|
||||
|
||||
class ObservedClassifier:
|
||||
prelude: Mapping[str, object] = {}
|
||||
|
||||
async def run(self, context: RoutingContext) -> RoutingContext:
|
||||
self.prelude = dict(context.metadata)
|
||||
return context
|
||||
|
||||
async def classify(self, context: RoutingContext) -> str:
|
||||
await litellm.acompletion(
|
||||
model="openai/gpt-5.6-luna", api_key="test", mock_response="SIMPLE",
|
||||
messages=[{"role": "user", "content": "choose a tier"}],
|
||||
metadata={**context.metadata, "internal_call_origin": "autorouter_classifier"},
|
||||
)
|
||||
return "SIMPLE"
|
||||
|
||||
class CallbackComplete(CustomLogger):
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
completed.set()
|
||||
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()])
|
||||
classifier: Final = ObservedClassifier()
|
||||
router: Final = Router(model_list=[
|
||||
{**self._marker("blue"), "litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "custom", "classifier_plugin": classifier,
|
||||
"tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False,
|
||||
},
|
||||
}},
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
|
||||
], plugins=[classifier])
|
||||
stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"}
|
||||
request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}}
|
||||
with logger.tracer.start_as_current_span("request") as root:
|
||||
set_request_root_span(root)
|
||||
await router.async_get_available_deployment(
|
||||
model="traced-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}]
|
||||
)
|
||||
await completed.wait()
|
||||
classifier_span: Final = next(
|
||||
span for span in exporter.get_finished_spans()
|
||||
if (span.attributes or {}).get("litellm.request.purpose") == "autorouter_classifier"
|
||||
)
|
||||
assert not any(key.startswith("litellm.routing.") for key in classifier_span.attributes)
|
||||
assert "routing_decision" not in classifier.prelude
|
||||
assert "_autorouter_baseline_route" not in classifier.prelude
|
||||
assert classifier.prelude["user_api_key_team_id"] == "team"
|
||||
assert "routing_decision" not in request["metadata"]
|
||||
assert "_autorouter_baseline_route" not in request["metadata"]
|
||||
assert request["litellm_metadata"]["routing_decision"]["cause"] == "classifier_plugin"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("build_fails", [False, True])
|
||||
async def test_semantic_internal_failure_keeps_phase_identity_without_parent_retry_attribution(
|
||||
self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter, build_fails: bool,
|
||||
) -> None:
|
||||
logger, exporter = _tracing
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
query_statuses: Final = iter((503, 200))
|
||||
|
||||
def embedding_response(request: httpx.Request) -> httpx.Response:
|
||||
status: Final = 400 if build_fails else (
|
||||
next(query_statuses) if json.loads(request.content)["input"] == ["query-retry"] else 200
|
||||
)
|
||||
return httpx.Response(status, headers={"x-should-retry": "false"}, json={
|
||||
"object": "list", "model": "text-embedding-3-small",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
})
|
||||
|
||||
upstream: Final = respx_mock.post("https://embedding.example/v1/embeddings").mock(side_effect=embedding_response)
|
||||
router: Final = Router(model_list=[
|
||||
{"model_name": "semantic", "model_info": {"id": "semantic-definition"}, "litellm_params": {
|
||||
"model": "auto_router/semantic-router", "auto_router_default_model": "answer",
|
||||
"auto_router_embedding_model": "embed",
|
||||
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
|
||||
}},
|
||||
*({"model_name": "embed", "model_info": {"id": f"embed-{index}"}, "litellm_params": {
|
||||
"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1",
|
||||
"api_key": "test", "max_retries": 0,
|
||||
}} for index in range(2)),
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
|
||||
], num_retries=1, retry_after=0)
|
||||
stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"}
|
||||
request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale}}
|
||||
with logger.tracer.start_as_current_span("request") as root:
|
||||
set_request_root_span(root)
|
||||
call: Final = router.async_get_available_deployment(
|
||||
model="semantic", request_kwargs=request, messages=[{"role": "user", "content": "query-retry"}]
|
||||
)
|
||||
if build_fails:
|
||||
with pytest.raises(ValueError, match="API call failed"):
|
||||
await call
|
||||
else:
|
||||
await call
|
||||
spans: Final = {span.name: span for span in exporter.get_finished_spans()}
|
||||
assert spans["route semantic"].attributes["litellm.routing.router_config_id"] == "semantic-definition"
|
||||
assert not any(event.name == "litellm.routing.retry" for event in spans["request"].events)
|
||||
if build_fails:
|
||||
assert all(
|
||||
"routing_decision" not in bucket and "_autorouter_baseline_route" not in bucket
|
||||
for bucket in (request["metadata"], request["litellm_metadata"])
|
||||
)
|
||||
else:
|
||||
assert sum(json.loads(call.request.content)["input"] == ["query-retry"] for call in upstream.calls) == 2
|
||||
assert request["litellm_metadata"]["routing_decision"]["cause"] == "semantic_match"
|
||||
assert sum(span.name == "route embed" for span in exporter.get_finished_spans()) == 2
|
||||
assert any(event.name == "litellm.request.deployment_selected" for event in spans["request"].events)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_alias_and_tags_across_strategy_families_keep_selected_definition(
|
||||
self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]"
|
||||
) -> None:
|
||||
logger, exporter = _tracing
|
||||
router: Final = Router(model_list=[
|
||||
self._marker("blue"),
|
||||
{
|
||||
"model_name": "traced-router",
|
||||
"model_info": {"id": "definition-semantic"},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/semantic-router",
|
||||
"tags": ["blue"],
|
||||
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
|
||||
"auto_router_default_model": "answer",
|
||||
"auto_router_embedding_model": "embed",
|
||||
},
|
||||
},
|
||||
])
|
||||
with logger.start_phase_span("route traced-router"):
|
||||
response: Final = await router.async_pre_routing_hook(
|
||||
model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}}
|
||||
)
|
||||
assert response is None
|
||||
span: Final = next(span for span in exporter.get_finished_spans() if span.name == "route traced-router")
|
||||
assert span.attributes is not None
|
||||
assert span.attributes["litellm.routing.router_config_id"] == router.model_list[1]["model_info"]["id"]
|
||||
assert span.attributes["litellm.routing.router_type"] == "semantic"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_team_alias_keeps_registered_identity_after_routing_and_retry(
|
||||
self,
|
||||
_tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]",
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
logger, exporter = _tracing
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={
|
||||
"object": "list",
|
||||
"model": "text-embedding-3-small",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
})
|
||||
router: Final = Router(model_list=[
|
||||
{
|
||||
"model_name": "internal-semantic-router",
|
||||
"model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/semantic-router",
|
||||
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
|
||||
"auto_router_default_model": "answer",
|
||||
"auto_router_embedding_model": "embed",
|
||||
},
|
||||
},
|
||||
{"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}},
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
|
||||
])
|
||||
request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}}
|
||||
with logger.tracer.start_as_current_span("request"):
|
||||
await router.async_get_available_deployment(
|
||||
model="public-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}]
|
||||
)
|
||||
router.log_retry({**request, "model": "answer"}, ValueError("retry"))
|
||||
decision: Final = request["metadata"]["routing_decision"]
|
||||
assert decision["router_model_name"] == router.model_list[0]["model_name"]
|
||||
assert decision["router_config_id"] == router.model_list[0]["model_info"]["id"]
|
||||
assert decision["cause"] == "semantic_match"
|
||||
spans: Final = {span.name: span for span in exporter.get_finished_spans()}
|
||||
assert spans["route public-router"].attributes["litellm.routing.router_model_name"] == decision["router_model_name"]
|
||||
assert spans["route public-router"].attributes["litellm.routing.router_config_id"] == decision["router_config_id"]
|
||||
retry: Final = next(event for event in spans["request"].events if event.name == "litellm.routing.retry")
|
||||
assert retry.attributes["litellm.routing.router_model_name"] == decision["router_model_name"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("passthrough", [False, True])
|
||||
async def test_attempt_spans_keep_selected_config_and_plain_fallback_clears_it(
|
||||
self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", passthrough: bool
|
||||
) -> None:
|
||||
logger, exporter = _tracing
|
||||
router: Final = Router(model_list=[
|
||||
self._marker("blue"), self._marker("green"),
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test", "use_in_pass_through": True, "tags": ["blue", "green"]}},
|
||||
], enable_tag_filtering=True)
|
||||
request: Final = {"metadata": {"tags": ["green"]}, "litellm_metadata": {"tags": ["green"]}}
|
||||
pick: Final = router.async_get_available_deployment_for_pass_through if passthrough else router.async_get_available_deployment
|
||||
with logger.tracer.start_as_current_span("request"):
|
||||
await pick(model="traced-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}])
|
||||
decision: Final = request["litellm_metadata"]["routing_decision"]
|
||||
assert decision["router_config_id"] == "definition-green"
|
||||
assert decision["router_config_updated_at"] == "2026-10-01T12:00:00+00:00"
|
||||
assert decision["router_config_fingerprint"] == router.complexity_routers["traced-router"][1].definition_fingerprint
|
||||
assert "routing_decision" not in request["metadata"]
|
||||
router.log_retry({**request, "model": "answer"}, ValueError("private-error-text"))
|
||||
await pick(model="answer", request_kwargs=request, messages=[{"role": "user", "content": "hello"}])
|
||||
assert "routing_decision" not in request["litellm_metadata"]
|
||||
spans: Final = {span.name: span for span in exporter.get_finished_spans()}
|
||||
assert spans["route traced-router"].attributes["litellm.routing.router_config_id"] == "definition-green"
|
||||
assert spans["route traced-router"].attributes["litellm.routing.routed_model"] == "answer"
|
||||
assert not any(key.startswith("litellm.routing.") for key in spans["route answer"].attributes)
|
||||
retry: Final = next(event for event in spans["request"].events if event.name == "litellm.routing.retry")
|
||||
assert retry.attributes["error.type"] == "ValueError"
|
||||
assert retry.attributes["litellm.retry.count"] == 1
|
||||
assert "private-error-text" not in str(retry.attributes)
|
||||
|
|
|
|||
17
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
17
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -45475,7 +45475,7 @@ export interface components {
|
|||
* Cause
|
||||
* @enum {string}
|
||||
*/
|
||||
cause?: "prompt_cache_cost" | "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "capability_classifier" | "jev_classifier" | "llm_v2_classifier" | "llm_v2_fallback" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "capability_classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "health_default_fallback" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit";
|
||||
cause?: "prompt_cache_cost" | "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "capability_classifier" | "jev_classifier" | "llm_v2_classifier" | "llm_v2_fallback" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "capability_classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "health_default_fallback" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit" | "semantic_match" | "semantic_no_match" | "semantic_error";
|
||||
/** Classifier Calibrated Capable P Solve */
|
||||
classifier_calibrated_capable_p_solve?: number;
|
||||
/** Classifier Calibrated Efficient P Solve */
|
||||
|
|
@ -45496,6 +45496,13 @@ export interface components {
|
|||
classifier_crux?: string;
|
||||
/** Classifier Efficient P Solve */
|
||||
classifier_efficient_p_solve?: number;
|
||||
/** Classifier Error Type */
|
||||
classifier_error_type?: string;
|
||||
/**
|
||||
* Classifier Failure Reason
|
||||
* @enum {string}
|
||||
*/
|
||||
classifier_failure_reason?: "timeout" | "circuit_open" | "not_configured" | "unsupported_input" | "invalid_response" | "declined" | "classifier_error";
|
||||
/** Classifier Max Quality Gap */
|
||||
classifier_max_quality_gap?: number;
|
||||
/** Classifier Model */
|
||||
|
|
@ -45531,13 +45538,19 @@ export interface components {
|
|||
request_type?: string;
|
||||
/** Routed Model */
|
||||
routed_model?: string;
|
||||
/** Router Config Fingerprint */
|
||||
router_config_fingerprint?: string;
|
||||
/** Router Config Id */
|
||||
router_config_id?: string;
|
||||
/** Router Config Updated At */
|
||||
router_config_updated_at?: string;
|
||||
/** Router Model Name */
|
||||
router_model_name?: string;
|
||||
/**
|
||||
* Router Type
|
||||
* @enum {string}
|
||||
*/
|
||||
router_type?: "complexity" | "adaptive" | "quality";
|
||||
router_type?: "complexity" | "adaptive" | "quality" | "semantic";
|
||||
/** Savings Baseline Deployment Id */
|
||||
savings_baseline_deployment_id?: string;
|
||||
/** Savings Baseline Model */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue