From b83a1f0608835ecba4b199155e1a585711477840 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 7 Nov 2026 15:52:00 -0800 Subject: [PATCH] fix(otel): name the routed model on auto-router LLM spans (#50498) On an auto-router request the LLM span reported the name the caller asked for (the router or an alias of it) as gen_ai.request.model (OTel v2) and gen_ai.response.model (v1 and v2). Both now name the model group the router picked, read from the request's routing_decision. In v2 the span name and the vendor keys built from the request model (Langfuse, OpenInference, Langtrace, Weave) follow. Requests that were not auto-routed are unchanged. --- litellm/integrations/opentelemetry.py | 8 +- litellm/integrations/otel/model/metadata.py | 24 ++- litellm/integrations/otel/routing.py | 14 +- .../test_otel_auto_router_model.py | 180 ++++++++++++++++++ .../otel/test_otel_v2_sources_of_truth.py | 21 ++ tests/unit/integrations/test_opentelemetry.py | 29 +++ 6 files changed, 264 insertions(+), 12 deletions(-) create mode 100644 tests/integration/observability/test_otel_auto_router_model.py diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 5cd201a2f47..9bec220e073 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -34,7 +34,7 @@ from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls -from litellm.integrations.otel.routing import routing_decision_attributes +from litellm.integrations.otel.routing import routed_model_name, routing_decision_attributes from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -2617,7 +2617,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) # The model used to generate the response. - response_model: Final = _response_field(response_obj, "model") + response_model: Final = routed_model_name( + _response_field(response_obj, "model"), + standard_logging_payload.get("model_group"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped + decision, + ) if response_model: self.safe_set_attribute( span=span, diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 28d6ad54607..f9739a4ba15 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -29,9 +29,10 @@ asks for a *model group* (e.g. ``gpt-4o``) that routes to a concrete deployment * ``hidden_params.litellm_model_name`` — a secondary source for the dispatched model (populated only on some call paths, e.g. files). -So ``gen_ai.request.model`` is the *group* (falling back to the call model on the -SDK path, which has no group), and ``litellm.provider.model`` is the *dispatched* -model. They coincide on the SDK path, which is correct. +So ``gen_ai.request.model`` is the *group* (the group an auto router picked, when +one routed the call; the call model on the SDK path, which has no group), and +``litellm.provider.model`` is the *dispatched* model. They coincide on the SDK +path, which is correct. """ from __future__ import annotations @@ -52,6 +53,7 @@ from litellm.constants import ( from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY, GenAIOperation, resolve_operation from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds +from litellm.integrations.otel.routing import routed_model_name from litellm.types.utils import InternalCallOrigin if TYPE_CHECKING: @@ -170,7 +172,8 @@ class RequestIdentity: class RequestContext: """The fully-resolved view of a closed request, parsed once from the payload. - ``request_model`` is the user-facing requested model and ``provider_model`` + ``request_model`` is the user-facing requested model (or the group an auto + router picked for it) and ``provider_model`` (on :attr:`identity`) is the model litellm dispatched to the provider; the two differ on the proxy (group vs. deployment) and coincide on the SDK path. """ @@ -194,12 +197,15 @@ class RequestContext: raw_response: Final = payload.get("response") response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}) model_group: Final = as_str(payload.get("model_group")) or as_str(raw_meta.get("model_group")) + decision: Final = raw_meta.get("routing_decision") return cls( - # The user asked for the group; fall back to the call model on the SDK - # path, which has no group. Empty string (never None) so the span name - # builder and the mapper see a plain string. - request_model=model_group or as_str(payload.get("model")) or "", - response_model=as_str(response.get("model")), + # The user asked for the group, or an auto router picked one for it (the + # proxy restamps the response model to the requested name, so that gets + # the pick too); fall back to the call model on the SDK path, which has no + # group. Empty string (never None) so the span name builder and the mapper + # see a plain string. + request_model=routed_model_name(model_group, model_group, decision) or as_str(payload.get("model")) or "", + response_model=routed_model_name(as_str(response.get("model")), model_group, decision), model_group=model_group, model_id=as_str(payload.get("model_id")) or _model_info_id(raw_meta.get("model_info")), api_base=as_str(payload.get("api_base")) or as_str(hidden.get("api_base")), diff --git a/litellm/integrations/otel/routing.py b/litellm/integrations/otel/routing.py index 339775855be..1143e6d2dbe 100644 --- a/litellm/integrations/otel/routing.py +++ b/litellm/integrations/otel/routing.py @@ -1,8 +1,11 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Final, TypeAlias +from typing import Final, TypeAlias, TypeVar + +from litellm.litellm_core_utils.core_helpers import as_str_mapping RoutingAttributeValue: TypeAlias = str | int | float | bool +_Model = TypeVar("_Model") # rebind-ok: a TypeVar cannot be Final _ROUTING_SCALAR_FIELDS: Final = frozenset( { @@ -60,3 +63,12 @@ def routing_decision_attributes(decision: Mapping[str, object] | None) -> Mappin if isinstance(value := decision.get(key), (str, int, float, bool)) } ) + + +def routed_model_name(model: _Model, requested_model: object, decision: object) -> _Model | str: + """The model an auto router picked when ``model`` is only the name the caller requested, else ``model``.""" + fields: Final = as_str_mapping(decision) + routed: Final = fields.get("routed_model") if fields is not None else None + if model and model == requested_model and isinstance(routed, str) and routed: + return routed + return model diff --git a/tests/integration/observability/test_otel_auto_router_model.py b/tests/integration/observability/test_otel_auto_router_model.py new file mode 100644 index 00000000000..d056112daca --- /dev/null +++ b/tests/integration/observability/test_otel_auto_router_model.py @@ -0,0 +1,180 @@ +import json +import re +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.otlp_sink import SpanSinks, recorded_spans +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(300) + +MARKER: Final = re.compile(rb"autorouter-[0-9a-f]{32}") +TIERS: Final = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") +USAGE: Final = {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6} + + +@dataclass(frozen=True, slots=True) +class AutoRouterRig: + proxy: Gateway + sink: str + router: str + alias: str + routed: str + + +def _upstream(request: Request) -> Reply: + found: Final = MARKER.search(request.body) + if found is None: + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + if request.target.endswith("/responses"): + output: Final = { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + return Reply( + body=json.dumps( + { + "id": "resp_" + found.group(0).decode(), + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini-2024-07-18", + "output": [output], + "usage": {"input_tokens": 5, "output_tokens": 1, "total_tokens": 6}, + } + ).encode() + ) + head: Final = {"id": "chatcmpl-" + found.group(0).decode(), "created": 1, "model": "gpt-4o-mini-2024-07-18"} + if json.loads(request.body).get("stream") is not True: + message: Final = {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + return Reply( + body=json.dumps({**head, "object": "chat.completion", "choices": [message], "usage": USAGE}).encode() + ) + chunks: Final = ( + {**head, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "ok"}}]}, + { + **head, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": USAGE, + }, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(f"data: {json.dumps(chunk)}\n\n".encode() for chunk in chunks), b"data: [DONE]\n\n"), + ) + + +@pytest.fixture(scope="module") +def gateway(audit_sinks: SpanSinks) -> Iterator[Gateway]: + with gateway_from_environment() as base: + yield base + + +@pytest.fixture(scope="module", params=[False, True], ids=["otel-v1", "otel-v2"]) +def rig( + request: pytest.FixtureRequest, + gateway: Gateway, + audit_sinks: SpanSinks, + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[AutoRouterRig]: + otel_v2: Final = request.param is True + router: Final = "auto-router-" + uuid.uuid4().hex[:8] + alias: Final = "smart-" + uuid.uuid4().hex[:8] + routed: Final = "tier-model-" + uuid.uuid4().hex[:8] + directory: Final = tmp_path_factory.mktemp("auto-router-otel") + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + with wire_server(_upstream) as provider: + config["litellm_settings"] = {**config.get("litellm_settings", {}), "callbacks": ["otel"]} + config["router_settings"] = {**config.get("router_settings", {}), "model_group_alias": {alias: router}} + config["callback_settings"] = { + "otel": {"exporter": "http/json", "endpoint": audit_sinks.operator, "use_simple_processor": True} + } + config["model_list"] = [ + { + "model_name": routed, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": provider.url, "api_key": "synthetic"}, + }, + { + "model_name": router, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": dict.fromkeys(TIERS, routed)}, + }, + }, + ] + path: Final = directory / "auto_router_otel.yaml" + path.write_text(yaml.safe_dump(config)) + overrides: Final = { + "OTEL_EXPORTER": "http/json", + "OTEL_ENDPOINT": audit_sinks.operator, + **({"LITELLM_OTEL_V2": "1"} if otel_v2 else {}), + } + with owned_proxy( + gateway, + directory, + overrides, + config=path, + remove_environment=() if otel_v2 else ("LITELLM_OTEL_V2",), + ) as proxy: + yield AutoRouterRig(proxy=proxy, sink=audit_sinks.operator, router=router, alias=alias, routed=routed) + + +@pytest.mark.parametrize( + ("route", "stream", "via_alias"), + [ + ("/v1/chat/completions", False, False), + ("/v1/chat/completions", True, False), + ("/v1/messages", False, False), + ("/v1/responses", False, False), + ("/v1/chat/completions", False, True), + ], + ids=["chat", "chat-stream", "messages", "responses", "chat-via-alias"], +) +def test_auto_router_llm_span_names_the_routed_model_and_carries_the_decision( + rig: AutoRouterRig, route: str, stream: bool, via_alias: bool +) -> None: + prompt: Final = "hi autorouter-" + uuid.uuid4().hex + body: Final[dict[str, JsonValue]] = { + "model": rig.alias if via_alias else rig.router, + "stream": stream, + **({"input": prompt} if route == "/v1/responses" else {"messages": [{"role": "user", "content": prompt}]}), + **({"max_tokens": 16} if route == "/v1/messages" else {}), + } + start, _ = recorded_spans(rig.sink) + response: Final = rig.proxy.request("POST", route, body) + assert response.status_code == 200, response.text + call_id: Final = response.headers["x-litellm-call-id"] + + def llm_span() -> Mapping[str, JsonValue] | None: + return next( + ( + span["attributes"] + for span in recorded_spans(rig.sink, start)[1] + if span["attributes"].get("litellm.call_id") == call_id + and "litellm.routing.router_model_name" in span["attributes"] + ), + None, + ) + + attributes: Final = eventually(llm_span, lambda found: found is not None, seconds=60) + assert attributes is not None + shown: Final = json.dumps(attributes, sort_keys=True, default=str) + underlying: Final = frozenset({rig.routed, "gpt-4o-mini"}) + assert attributes["gen_ai.request.model"] in underlying, shown + assert attributes["gen_ai.response.model"] in underlying, shown + assert attributes["litellm.routing.router_model_name"] == rig.router, shown + assert attributes["litellm.routing.router_type"] == "complexity", shown + assert attributes["litellm.routing.routed_model"] == rig.routed, shown + assert attributes["litellm.routing.tier"] in TIERS, shown diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index c29da4607eb..3c95a85e9a8 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -539,6 +539,27 @@ def test_llm_call_adapter_extracts_all_fields(): assert data.identity.key_hash == "hsh" +@pytest.mark.parametrize( + ("model_group", "response_model", "expected"), + [ + ("auto-router", "auto-router", ("gpt-4o-group", "gpt-4o-group")), + ("smart", "smart", ("gpt-4o-group", "gpt-4o-group")), + ("auto-router", "gpt-4o-2024", ("gpt-4o-group", "gpt-4o-2024")), + (None, None, ("gpt-4o", None)), + ], + ids=["restamped-to-router", "restamped-to-alias", "provider-model", "no-group-no-response"], +) +def test_llm_call_adapter_names_the_routed_model_for_an_auto_router_call(model_group, response_model, expected): + data = LLMCallSpanData.from_standard_logging_payload( + _sample_payload( + model_group=model_group, + response={"id": "resp_1", "model": response_model, "choices": [{"finish_reason": "stop"}]}, + metadata={"routing_decision": {"routed_model": "gpt-4o-group"}}, + ) + ) + assert (data.request_model, data.response_model) == expected + + def test_llm_call_adapter_extracts_cache_tokens_from_usage_object(): payload = _sample_payload() payload["metadata"] = { diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index c4fd7ed51c7..4ef95d3a79d 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -372,6 +372,35 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada assert finished.attributes["metadata.routing_decision"] == str(decision) +@pytest.mark.parametrize( + ("decision", "expected"), + [({"router_model_name": "auto-router", "routed_model": "answer"}, "answer"), ("not-a-mapping", "auto-router")], + ids=["routed", "malformed-decision"], +) +def test_v1_auto_router_response_model_names_the_routed_model(decision: object, expected: str) -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "gpt-4o-mini", + "standard_logging_object": { + "id": "routing-call", + "call_type": "acompletion", + "model_group": "auto-router", + "metadata": {"routing_decision": decision}, + }, + }, + {"model": "auto-router", "choices": [], "usage": {"prompt_tokens": 3}}, + ) + (finished,) = exporter.get_finished_spans() + attributes: Final = finished.attributes + assert (attributes["gen_ai.response.model"], attributes["gen_ai.usage.input_tokens"]) == (expected, 3) + + def test_v1_server_tool_calls_lead_the_completion_function_call_attributes() -> None: exporter: Final = InMemorySpanExporter() provider: Final = TracerProvider()