fix(otel): name the routed model on auto-router LLM spans (#50498)

On an auto-router request the LLM span reported the name the caller asked
for (the router or an alias of it) as gen_ai.request.model (OTel v2) and
gen_ai.response.model (v1 and v2). Both now name the model group the router
picked, read from the request's routing_decision. In v2 the span name and the
vendor keys built from the request model (Langfuse, OpenInference, Langtrace,
Weave) follow. Requests that were not auto-routed are unchanged.

litellm/integrations/opentelemetry.py

@@ -34,7 +34,7 @@ from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.metadata import flatten_metadata
 from litellm.integrations.otel.model.semconv import LiteLLM, Metric
 from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
-from litellm.integrations.otel.routing import routing_decision_attributes
+from litellm.integrations.otel.routing import routed_model_name, routing_decision_attributes
 from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -2617,7 +2617,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                 )
 
             # The model used to generate the response.
-            response_model: Final = _response_field(response_obj, "model")
+            response_model: Final = routed_model_name(
+                _response_field(response_obj, "model"),
+                standard_logging_payload.get("model_group"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # kwargs is untyped
+                decision,
+            )
             if response_model:
                 self.safe_set_attribute(
                     span=span,

litellm/integrations/otel/model/metadata.py

@@ -29,9 +29,10 @@ asks for a *model group* (e.g. ``gpt-4o``) that routes to a concrete deployment
 * ``hidden_params.litellm_model_name`` — a secondary source for the dispatched
   model (populated only on some call paths, e.g. files).
 
-So ``gen_ai.request.model`` is the *group* (falling back to the call model on the
-SDK path, which has no group), and ``litellm.provider.model`` is the *dispatched*
-model. They coincide on the SDK path, which is correct.
+So ``gen_ai.request.model`` is the *group* (the group an auto router picked, when
+one routed the call; the call model on the SDK path, which has no group), and
+``litellm.provider.model`` is the *dispatched* model. They coincide on the SDK
+path, which is correct.
 """
 
 from __future__ import annotations
@@ -52,6 +53,7 @@ from litellm.constants import (
 from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY, GenAIOperation, resolve_operation
 from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls
 from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds
+from litellm.integrations.otel.routing import routed_model_name
 from litellm.types.utils import InternalCallOrigin
 
 if TYPE_CHECKING:
@@ -170,7 +172,8 @@ class RequestIdentity:
 class RequestContext:
     """The fully-resolved view of a closed request, parsed once from the payload.
 
-    ``request_model`` is the user-facing requested model and ``provider_model``
+    ``request_model`` is the user-facing requested model (or the group an auto
+    router picked for it) and ``provider_model``
     (on :attr:`identity`) is the model litellm dispatched to the provider; the two
     differ on the proxy (group vs. deployment) and coincide on the SDK path.
     """
@@ -194,12 +197,15 @@ class RequestContext:
         raw_response: Final = payload.get("response")
         response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {})
         model_group: Final = as_str(payload.get("model_group")) or as_str(raw_meta.get("model_group"))
+        decision: Final = raw_meta.get("routing_decision")
         return cls(
-            # The user asked for the group; fall back to the call model on the SDK
-            # path, which has no group. Empty string (never None) so the span name
-            # builder and the mapper see a plain string.
-            request_model=model_group or as_str(payload.get("model")) or "",
-            response_model=as_str(response.get("model")),
+            # The user asked for the group, or an auto router picked one for it (the
+            # proxy restamps the response model to the requested name, so that gets
+            # the pick too); fall back to the call model on the SDK path, which has no
+            # group. Empty string (never None) so the span name builder and the mapper
+            # see a plain string.
+            request_model=routed_model_name(model_group, model_group, decision) or as_str(payload.get("model")) or "",
+            response_model=routed_model_name(as_str(response.get("model")), model_group, decision),
             model_group=model_group,
             model_id=as_str(payload.get("model_id")) or _model_info_id(raw_meta.get("model_info")),
             api_base=as_str(payload.get("api_base")) or as_str(hidden.get("api_base")),

litellm/integrations/otel/routing.py

@@ -1,8 +1,11 @@
 from collections.abc import Mapping
 from types import MappingProxyType
-from typing import Final, TypeAlias
+from typing import Final, TypeAlias, TypeVar
+
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 
 RoutingAttributeValue: TypeAlias = str | int | float | bool
+_Model = TypeVar("_Model")  # rebind-ok: a TypeVar cannot be Final
 
 _ROUTING_SCALAR_FIELDS: Final = frozenset(
     {
@@ -60,3 +63,12 @@ def routing_decision_attributes(decision: Mapping[str, object] | None) -> Mappin
             if isinstance(value := decision.get(key), (str, int, float, bool))
         }
     )
+
+
+def routed_model_name(model: _Model, requested_model: object, decision: object) -> _Model | str:
+    """The model an auto router picked when ``model`` is only the name the caller requested, else ``model``."""
+    fields: Final = as_str_mapping(decision)
+    routed: Final = fields.get("routed_model") if fields is not None else None
+    if model and model == requested_model and isinstance(routed, str) and routed:
+        return routed
+    return model

tests/integration/observability/test_otel_auto_router_model.py

@@ -0,0 +1,180 @@
+import json
+import re
+import uuid
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+import pytest
+import yaml
+from integration._support.client import Gateway, eventually, gateway_from_environment
+from integration._support.otlp_sink import SpanSinks, recorded_spans
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, wire_server
+from pydantic import JsonValue
+
+pytestmark: Final = pytest.mark.timeout(300)
+
+MARKER: Final = re.compile(rb"autorouter-[0-9a-f]{32}")
+TIERS: Final = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")
+USAGE: Final = {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}
+
+
+@dataclass(frozen=True, slots=True)
+class AutoRouterRig:
+    proxy: Gateway
+    sink: str
+    router: str
+    alias: str
+    routed: str
+
+
+def _upstream(request: Request) -> Reply:
+    found: Final = MARKER.search(request.body)
+    if found is None:
+        return Reply(body=json.dumps({"object": "list", "data": []}).encode())
+    if request.target.endswith("/responses"):
+        output: Final = {
+            "type": "message",
+            "id": "msg_1",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": "ok", "annotations": []}],
+        }
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": "resp_" + found.group(0).decode(),
+                    "object": "response",
+                    "created_at": 1,
+                    "status": "completed",
+                    "model": "gpt-4o-mini-2024-07-18",
+                    "output": [output],
+                    "usage": {"input_tokens": 5, "output_tokens": 1, "total_tokens": 6},
+                }
+            ).encode()
+        )
+    head: Final = {"id": "chatcmpl-" + found.group(0).decode(), "created": 1, "model": "gpt-4o-mini-2024-07-18"}
+    if json.loads(request.body).get("stream") is not True:
+        message: Final = {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
+        return Reply(
+            body=json.dumps({**head, "object": "chat.completion", "choices": [message], "usage": USAGE}).encode()
+        )
+    chunks: Final = (
+        {**head, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "ok"}}]},
+        {
+            **head,
+            "object": "chat.completion.chunk",
+            "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+            "usage": USAGE,
+        },
+    )
+    return Reply(
+        content_type="text/event-stream",
+        chunks=(*(f"data: {json.dumps(chunk)}\n\n".encode() for chunk in chunks), b"data: [DONE]\n\n"),
+    )
+
+
+@pytest.fixture(scope="module")
+def gateway(audit_sinks: SpanSinks) -> Iterator[Gateway]:
+    with gateway_from_environment() as base:
+        yield base
+
+
+@pytest.fixture(scope="module", params=[False, True], ids=["otel-v1", "otel-v2"])
+def rig(
+    request: pytest.FixtureRequest,
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    tmp_path_factory: pytest.TempPathFactory,
+) -> Iterator[AutoRouterRig]:
+    otel_v2: Final = request.param is True
+    router: Final = "auto-router-" + uuid.uuid4().hex[:8]
+    alias: Final = "smart-" + uuid.uuid4().hex[:8]
+    routed: Final = "tier-model-" + uuid.uuid4().hex[:8]
+    directory: Final = tmp_path_factory.mktemp("auto-router-otel")
+    config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+    with wire_server(_upstream) as provider:
+        config["litellm_settings"] = {**config.get("litellm_settings", {}), "callbacks": ["otel"]}
+        config["router_settings"] = {**config.get("router_settings", {}), "model_group_alias": {alias: router}}
+        config["callback_settings"] = {
+            "otel": {"exporter": "http/json", "endpoint": audit_sinks.operator, "use_simple_processor": True}
+        }
+        config["model_list"] = [
+            {
+                "model_name": routed,
+                "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": provider.url, "api_key": "synthetic"},
+            },
+            {
+                "model_name": router,
+                "litellm_params": {
+                    "model": "auto_router/complexity_router",
+                    "complexity_router_config": {"tiers": dict.fromkeys(TIERS, routed)},
+                },
+            },
+        ]
+        path: Final = directory / "auto_router_otel.yaml"
+        path.write_text(yaml.safe_dump(config))
+        overrides: Final = {
+            "OTEL_EXPORTER": "http/json",
+            "OTEL_ENDPOINT": audit_sinks.operator,
+            **({"LITELLM_OTEL_V2": "1"} if otel_v2 else {}),
+        }
+        with owned_proxy(
+            gateway,
+            directory,
+            overrides,
+            config=path,
+            remove_environment=() if otel_v2 else ("LITELLM_OTEL_V2",),
+        ) as proxy:
+            yield AutoRouterRig(proxy=proxy, sink=audit_sinks.operator, router=router, alias=alias, routed=routed)
+
+
+@pytest.mark.parametrize(
+    ("route", "stream", "via_alias"),
+    [
+        ("/v1/chat/completions", False, False),
+        ("/v1/chat/completions", True, False),
+        ("/v1/messages", False, False),
+        ("/v1/responses", False, False),
+        ("/v1/chat/completions", False, True),
+    ],
+    ids=["chat", "chat-stream", "messages", "responses", "chat-via-alias"],
+)
+def test_auto_router_llm_span_names_the_routed_model_and_carries_the_decision(
+    rig: AutoRouterRig, route: str, stream: bool, via_alias: bool
+) -> None:
+    prompt: Final = "hi autorouter-" + uuid.uuid4().hex
+    body: Final[dict[str, JsonValue]] = {
+        "model": rig.alias if via_alias else rig.router,
+        "stream": stream,
+        **({"input": prompt} if route == "/v1/responses" else {"messages": [{"role": "user", "content": prompt}]}),
+        **({"max_tokens": 16} if route == "/v1/messages" else {}),
+    }
+    start, _ = recorded_spans(rig.sink)
+    response: Final = rig.proxy.request("POST", route, body)
+    assert response.status_code == 200, response.text
+    call_id: Final = response.headers["x-litellm-call-id"]
+
+    def llm_span() -> Mapping[str, JsonValue] | None:
+        return next(
+            (
+                span["attributes"]
+                for span in recorded_spans(rig.sink, start)[1]
+                if span["attributes"].get("litellm.call_id") == call_id
+                and "litellm.routing.router_model_name" in span["attributes"]
+            ),
+            None,
+        )
+
+    attributes: Final = eventually(llm_span, lambda found: found is not None, seconds=60)
+    assert attributes is not None
+    shown: Final = json.dumps(attributes, sort_keys=True, default=str)
+    underlying: Final = frozenset({rig.routed, "gpt-4o-mini"})
+    assert attributes["gen_ai.request.model"] in underlying, shown
+    assert attributes["gen_ai.response.model"] in underlying, shown
+    assert attributes["litellm.routing.router_model_name"] == rig.router, shown
+    assert attributes["litellm.routing.router_type"] == "complexity", shown
+    assert attributes["litellm.routing.routed_model"] == rig.routed, shown
+    assert attributes["litellm.routing.tier"] in TIERS, shown

tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py

@@ -539,6 +539,27 @@ def test_llm_call_adapter_extracts_all_fields():
     assert data.identity.key_hash == "hsh"
 
 
+@pytest.mark.parametrize(
+    ("model_group", "response_model", "expected"),
+    [
+        ("auto-router", "auto-router", ("gpt-4o-group", "gpt-4o-group")),
+        ("smart", "smart", ("gpt-4o-group", "gpt-4o-group")),
+        ("auto-router", "gpt-4o-2024", ("gpt-4o-group", "gpt-4o-2024")),
+        (None, None, ("gpt-4o", None)),
+    ],
+    ids=["restamped-to-router", "restamped-to-alias", "provider-model", "no-group-no-response"],
+)
+def test_llm_call_adapter_names_the_routed_model_for_an_auto_router_call(model_group, response_model, expected):
+    data = LLMCallSpanData.from_standard_logging_payload(
+        _sample_payload(
+            model_group=model_group,
+            response={"id": "resp_1", "model": response_model, "choices": [{"finish_reason": "stop"}]},
+            metadata={"routing_decision": {"routed_model": "gpt-4o-group"}},
+        )
+    )
+    assert (data.request_model, data.response_model) == expected
+
+
 def test_llm_call_adapter_extracts_cache_tokens_from_usage_object():
     payload = _sample_payload()
     payload["metadata"] = {

tests/unit/integrations/test_opentelemetry.py

@@ -372,6 +372,35 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada
     assert finished.attributes["metadata.routing_decision"] == str(decision)
 
 
+@pytest.mark.parametrize(
+    ("decision", "expected"),
+    [({"router_model_name": "auto-router", "routed_model": "answer"}, "answer"), ("not-a-mapping", "auto-router")],
+    ids=["routed", "malformed-decision"],
+)
+def test_v1_auto_router_response_model_names_the_routed_model(decision: object, expected: str) -> None:
+    exporter: Final = InMemorySpanExporter()
+    provider: Final = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    otel: Final = OpenTelemetry(tracer_provider=provider)
+    with provider.get_tracer(__name__).start_as_current_span("inference") as span:
+        otel.set_attributes(
+            span,
+            {
+                "model": "gpt-4o-mini",
+                "standard_logging_object": {
+                    "id": "routing-call",
+                    "call_type": "acompletion",
+                    "model_group": "auto-router",
+                    "metadata": {"routing_decision": decision},
+                },
+            },
+            {"model": "auto-router", "choices": [], "usage": {"prompt_tokens": 3}},
+        )
+    (finished,) = exporter.get_finished_spans()
+    attributes: Final = finished.attributes
+    assert (attributes["gen_ai.response.model"], attributes["gen_ai.usage.input_tokens"]) == (expected, 3)
+
+
 def test_v1_server_tool_calls_lead_the_completion_function_call_attributes() -> None:
     exporter: Final = InMemorySpanExporter()
     provider: Final = TracerProvider()