fix(otel): label pass-through operations from upstream header or route instead of chat

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-10-27 23:42:25 +00:00
parent cedae29cc7
commit bb74bb980e
14 changed files with 368 additions and 15 deletions

View file

@ -36,6 +36,10 @@ from litellm.integrations.otel.model.semconv import LiteLLM, Metric
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
from litellm.integrations.otel.routing import routing_decision_attributes
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.passthrough_operation import (
PASSTHROUGH_OPERATION_DATA,
passthrough_header_operation,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.service_tier_utils import (
@ -1707,9 +1711,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
provider: Final = _provider_label(params.get("custom_llm_provider"))
common_attrs = {
"gen_ai.operation.name": (
self._gen_ai_operation_name(kwargs) if self._gen_ai_semconv_latest_experimental else "chat"
),
"gen_ai.operation.name": self._gen_ai_operation_name(kwargs),
"gen_ai.system": provider,
"gen_ai.request.model": kwargs.get("model"),
"gen_ai.framework": "litellm",
@ -2687,14 +2689,23 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=safe_dumps(transformed_system_instructions),
)
if self._gen_ai_semconv_latest_experimental:
operation_name = self._gen_ai_operation_name(kwargs)
else:
operation_name = (
operation_name: Final = (
self._gen_ai_operation_name(kwargs)
if self._gen_ai_semconv_latest_experimental
else (
passthrough_header_operation(PASSTHROUGH_OPERATION_DATA.validate_python(kwargs))
or passthrough_header_operation(
PASSTHROUGH_OPERATION_DATA.validate_python(standard_logging_payload)
)
or "pass_through_endpoint"
)
if standard_logging_payload["call_type"] == "pass_through_endpoint"
else (
"chat"
if standard_logging_payload.get("call_type") == "completion"
else standard_logging_payload.get("call_type") or "chat"
)
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_OPERATION_NAME.value,

View file

@ -35,6 +35,10 @@ from enum import Enum
from typing import TYPE_CHECKING, Final, Protocol
from litellm.integrations.otel.mappers.utils import drop_none
from litellm.litellm_core_utils.passthrough_operation import (
PASSTHROUGH_OPERATION_DATA,
resolve_passthrough_operation,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
if TYPE_CHECKING:
@ -151,6 +155,8 @@ class OTELGenAISemconvMixin:
``chat``.
"""
call_type: Final = kwargs.get("call_type", "") or ""
if call_type == "pass_through_endpoint":
return resolve_passthrough_operation(PASSTHROUGH_OPERATION_DATA.validate_python(kwargs))
match call_type:
case s if "embedding" in s:
return "embeddings"

View file

@ -266,7 +266,7 @@ class LLMCallEvent:
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
raw_payload: Final = kwargs.get("standard_logging_object")
payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None
operation: Final = resolve_operation(as_str(kwargs.get("call_type")))
operation: Final = resolve_operation(as_str(kwargs.get("call_type")), kwargs)
model: Final = as_str(kwargs.get("model")) or ""
trace: Final = caller_trace_controls(kwargs)
return cls(

View file

@ -17,6 +17,7 @@ from litellm.integrations.otel.model.semconv import (
GenAIOperation,
GenAIOutputType,
MCPMethod,
UpstreamOperation,
resolve_operation,
resolve_output_type,
resolve_provider,
@ -405,7 +406,7 @@ class EmbeddingOutput:
@dataclass(frozen=True)
class LLMCallSpanData:
operation: GenAIOperation
operation: GenAIOperation | UpstreamOperation
provider: str
request_model: str
response_model: str | None
@ -479,7 +480,7 @@ class LLMCallSpanData:
# no prompt/response text.
finish_reasons: Final = _finish_reasons(choices_out)
call_type: Final = as_str(payload.get("call_type"))
operation: Final = resolve_operation(call_type)
operation: Final = resolve_operation(call_type, payload)
embedding_output: Final = (
EmbeddingOutput.from_response(response) if operation is GenAIOperation.EMBEDDINGS else None
)

View file

@ -4,11 +4,13 @@ without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
"""
from collections.abc import Mapping
from dataclasses import dataclass
from enum import Enum
from types import MappingProxyType
from typing import Final
from typing import Final, overload
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.passthrough_operation import resolve_passthrough_operation
class GenAIOperation(str, Enum):
@ -34,6 +36,7 @@ class GenAIOperation(str, Enum):
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
LITELLM_RESPONSES_MANAGEMENT = "litellm.responses_management"
LITELLM_MODERATION = "litellm.moderation"
LITELLM_PASS_THROUGH = "litellm.pass_through"
class GenAIOutputType(str, Enum):
@ -482,7 +485,22 @@ def resolve_provider(custom_llm_provider: str | None) -> str:
return mapped.value if mapped is not None else custom_llm_provider
def resolve_operation(call_type: str | None) -> GenAIOperation:
@dataclass(frozen=True, slots=True)
class UpstreamOperation:
value: str
@overload
def resolve_operation(call_type: str | None, data: None = None) -> GenAIOperation: ...
@overload
def resolve_operation(call_type: str | None, data: Mapping[str, object]) -> GenAIOperation | UpstreamOperation: ...
def resolve_operation(
call_type: str | None, data: Mapping[str, object] | None = None
) -> GenAIOperation | UpstreamOperation:
"""Map a litellm ``call_type`` to a ``gen_ai.operation.name`` value.
An unmapped call type still falls back to ``chat`` so every series keeps an
@ -490,6 +508,14 @@ def resolve_operation(call_type: str | None) -> GenAIOperation:
a new call type mislabelled as ``chat`` mixes its latency and cost into
everyone's chat charts, which is invisible until someone reads the numbers.
"""
if call_type == "pass_through_endpoint":
if data is None:
return GenAIOperation.LITELLM_PASS_THROUGH
passthrough: Final = resolve_passthrough_operation(data)
try:
return GenAIOperation(passthrough)
except ValueError:
return UpstreamOperation(passthrough)
if not call_type:
return GenAIOperation.CHAT
mapped: Final = _OPERATION_BY_CALL_TYPE.get(call_type.lower())

View file

@ -280,7 +280,7 @@ class GenAIMetricRecorder:
def _common_attributes(self, kwargs: Mapping[str, Any]) -> dict:
params: Final = kwargs.get("litellm_params") or {}
common_attrs: Final[dict] = {
GenAI.OPERATION_NAME: resolve_operation(kwargs.get("call_type")).value,
GenAI.OPERATION_NAME: resolve_operation(kwargs.get("call_type"), kwargs).value,
**_provider_attributes(params.get("custom_llm_provider")),
GenAI.REQUEST_MODEL: kwargs.get("model"),
"gen_ai.framework": "litellm",

View file

@ -120,6 +120,10 @@ from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages_async,
)
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.passthrough_operation import (
infer_passthrough_operation,
normalize_passthrough_operation,
)
from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request
from litellm.litellm_core_utils.redact_messages import (
redact_message_input_output_from_custom_logger,
@ -7564,6 +7568,16 @@ def get_standard_logging_object_payload(
litellm_params=litellm_params,
),
call_type=call_type or "",
passthrough_operation_name=(
normalize_passthrough_operation(logging_obj.model_call_details.get("passthrough_operation_name"))
if call_type == "pass_through_endpoint"
else None
),
passthrough_inferred_operation=(
infer_passthrough_operation(logging_obj.model_call_details)
if call_type == "pass_through_endpoint"
else None
),
cache_hit=cache_hit,
stream=stream,
status=status,

View file

@ -0,0 +1,59 @@
import re
from collections.abc import Mapping
from typing import Final
from urllib.parse import urlsplit
from pydantic import TypeAdapter, ValidationError
PASSTHROUGH_OPERATION_DATA: Final = TypeAdapter(Mapping[str, object])
def normalize_passthrough_operation(value: object) -> str | None:
if not isinstance(value, str):
return None
normalized: Final = value.strip().lower()
return normalized if re.fullmatch(r"[a-z][a-z0-9_.]{0,63}", normalized) else None
def _mapping(value: object) -> Mapping[str, object]:
try:
return PASSTHROUGH_OPERATION_DATA.validate_python(value)
except ValidationError:
return {}
def passthrough_header_operation(data: Mapping[str, object]) -> str | None:
return normalize_passthrough_operation(data.get("passthrough_operation_name"))
def infer_passthrough_operation(data: Mapping[str, object]) -> str:
inferred: Final = data.get("passthrough_inferred_operation")
if isinstance(inferred, str) and inferred in ("chat", "embeddings", "text_completion", "litellm.pass_through"):
return inferred
url: Final = _mapping(data.get("passthrough_logging_payload")).get("url")
try:
path: Final = urlsplit(url if isinstance(url, str) else "").path.lower()
except ValueError:
return "litellm.pass_through"
if path.endswith(
(
"/chat/completions",
"/v1/messages",
"/responses",
":generatecontent",
":streamgeneratecontent",
"/converse",
"/converse-stream",
"/chat",
)
):
return "chat"
if path.endswith(("/embeddings", ":embedcontent", ":batchembedcontents", "/embed")):
return "embeddings"
if path.endswith("/completions"):
return "text_completion"
return "litellm.pass_through"
def resolve_passthrough_operation(data: Mapping[str, object]) -> str:
return passthrough_header_operation(data) or infer_passthrough_operation(data)

View file

@ -150,6 +150,7 @@ from .streaming_handler import PassThroughStreamingHandler
from .success_handler import PassThroughEndpointLogging
from .upstream_usage_headers import (
UpstreamReportedUsage,
apply_upstream_operation_name,
apply_upstream_reported_usage,
)
@ -1539,6 +1540,7 @@ async def pass_through_request(
response = await async_client.send(req, stream=stream)
apply_upstream_operation_name(logging_obj, response.headers)
upstream_usage = apply_upstream_reported_usage(
logging_obj=logging_obj,
headers=response.headers,
@ -1627,6 +1629,7 @@ async def pass_through_request(
)
verbose_proxy_logger.debug("response.headers= %s", response.headers)
apply_upstream_operation_name(logging_obj, response.headers)
upstream_usage = apply_upstream_reported_usage(
logging_obj=logging_obj,
headers=response.headers,

View file

@ -15,10 +15,28 @@ import httpx
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.passthrough_operation import normalize_passthrough_operation
from litellm.types.utils import Usage
UPSTREAM_RESPONSE_COST_HEADER: Final = "x-litellm-response-cost"
UPSTREAM_TOTAL_TOKENS_HEADER: Final = "x-litellm-total-tokens"
UPSTREAM_OPERATION_NAME_HEADER: Final = "x-litellm-operation-name"
def apply_upstream_operation_name(logging_obj: LiteLLMLoggingObj, headers: httpx.Headers) -> None:
raw: Final = headers.get(UPSTREAM_OPERATION_NAME_HEADER)
if raw is None:
return
operation: Final = normalize_passthrough_operation(raw)
if operation is None:
verbose_proxy_logger.warning(
"pass_through_endpoint: upstream sent invalid %s=%r; ignoring operation name",
UPSTREAM_OPERATION_NAME_HEADER,
raw,
)
return
logging_obj.model_call_details["passthrough_operation_name"] = operation # rebind-ok: request callback state
# model_call_details key holding what the upstream reported, so later stages of
# the success path can tell an upstream-reported cost apart from one LiteLLM

View file

@ -3789,6 +3789,8 @@ class ClassifierAudit(TypedDict, total=False):
class StandardLoggingPayload(ClassifierAudit):
passthrough_operation_name: NotRequired[ReadOnly[str | None]]
passthrough_inferred_operation: NotRequired[ReadOnly[str | None]]
id: str
trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id

View file

@ -4,7 +4,7 @@ and the typed StandardLoggingPayload adapter. These need no OTel SDK."""
import json
import logging
import re
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from pathlib import Path
from typing import Final
@ -47,6 +47,11 @@ from litellm.integrations.otel.model.spans import (
validate_registry,
)
from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls
from litellm.integrations.otel.plumbing.metrics import GenAIMetricRecorder, create_genai_metrics
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from datetime import datetime
from itertools import chain
@pytest.fixture(autouse=True)
@ -148,6 +153,99 @@ def test_llm_call_span_name():
assert spans_mod.llm_call_span_name(data) == "chat gpt-4o"
@pytest.mark.parametrize(
("header", "path", "inferred", "expected"),
[
("retrieval", "/embeddings", "embeddings", "retrieval"),
(" EMBEDDINGS ", "/v1/messages", "chat", "embeddings"),
("vendor.custom", "/responses", "chat", "vendor.custom"),
(None, "/embeddings", "embeddings", "embeddings"),
("bad/value", "/v1/messages", "chat", "chat"),
(None, "/completions", "text_completion", "text_completion"),
(None, "/unknown", "litellm.pass_through", "litellm.pass_through"),
(None, "", "litellm.pass_through", "litellm.pass_through"),
*[
(None, path, "chat", "chat")
for path in (
"/chat/completions",
"/responses",
"/models/x:generateContent",
"/models/x:streamGenerateContent",
"/converse",
"/converse-stream",
"/chat",
)
],
*[
(None, path, "embeddings", "embeddings")
for path in ("/models/x:embedContent", "/models/x:batchEmbedContents", "/embed")
],
],
)
def test_passthrough_operation_agrees_in_v2_span_name_attributes_and_metrics(
header: str | None, path: str, inferred: str, expected: str
) -> None:
payload: Final = _sample_payload(
call_type="pass_through_endpoint",
passthrough_operation_name=header,
passthrough_inferred_operation=inferred,
)
kwargs: Final = {
"call_type": "pass_through_endpoint",
"model": "gpt-4o",
"passthrough_operation_name": header,
"passthrough_logging_payload": {"url": f"https://upstream{path}"},
"standard_logging_object": payload,
}
data: Final = LLMCallSpanData.from_standard_logging_payload(payload)
assert GenAIMapper().map(data)[GenAI.OPERATION_NAME] == expected
assert spans_mod.llm_call_span_name(data) == f"{expected} gpt-4o"
assert LLMCallEvent.from_dict(kwargs).provisional_span_name == f"{expected} gpt-4o"
reader: Final = InMemoryMetricReader()
provider: Final = MeterProvider(metric_readers=[reader])
recorder: Final = GenAIMetricRecorder(create_genai_metrics(provider.get_meter(__name__)))
now: Final = datetime(2026, 1, 1)
recorder.record(kwargs, None, now, now)
metrics: Final = reader.get_metrics_data()
assert metrics is not None
scopes: Final = chain.from_iterable(resource.scope_metrics for resource in metrics.resource_metrics)
instruments: Final = chain.from_iterable(scope.metrics for scope in scopes)
points: Final = chain.from_iterable(metric.data.data_points for metric in instruments)
operations: Final = tuple(point.attributes[GenAI.OPERATION_NAME] for point in points)
assert operations
assert set(operations) == {expected}
provider.shutdown()
@pytest.mark.parametrize(
"call_type",
["acompletion", "aembedding", "text_completion", "allm_passthrough_route", "passthrough_image_generation"],
)
def test_upstream_operation_does_not_override_other_v2_call_types(call_type: str) -> None:
baseline: Final = LLMCallSpanData.from_standard_logging_payload(_sample_payload(call_type=call_type))
overridden: Final = LLMCallSpanData.from_standard_logging_payload(
_sample_payload(
call_type=call_type, passthrough_operation_name="retrieval", passthrough_inferred_operation="embeddings"
)
)
assert GenAIMapper().map(overridden) == GenAIMapper().map(baseline)
assert spans_mod.llm_call_span_name(overridden) == spans_mod.llm_call_span_name(baseline)
def test_non_passthrough_operation_does_not_read_upstream_data() -> None:
class UnreadableOperationData(Mapping[str, object]):
def __getitem__(self, key: str) -> object:
raise AssertionError("non-pass-through calls must not read upstream data")
def __iter__(self) -> Iterator[str]:
raise AssertionError("non-pass-through calls must not copy upstream data")
def __len__(self) -> int:
raise AssertionError("non-pass-through calls must not inspect upstream data")
assert resolve_operation("acompletion", UnreadableOperationData()) is GenAIOperation.CHAT
# --- semconv (source of truth #1) ------------------------------------------- #

View file

@ -13,6 +13,7 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import MappingProxyType
from typing import Final
from itertools import chain
from unittest.mock import MagicMock, patch
import pytest
@ -59,6 +60,72 @@ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.utils import ChatCompletionMessageToolCall, Function
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
@pytest.mark.parametrize("experimental", [False, True])
@pytest.mark.parametrize(
("call_type", "header", "path", "resolved", "legacy"),
[
("pass_through_endpoint", "retrieval", "/embeddings", "retrieval", "retrieval"),
("pass_through_endpoint", " EMBEDDINGS ", "/v1/messages", "embeddings", "embeddings"),
("pass_through_endpoint", None, "/embeddings", "embeddings", "pass_through_endpoint"),
("pass_through_endpoint", "bad/value", "/v1/messages", "chat", "pass_through_endpoint"),
("pass_through_endpoint", None, "/completions", "text_completion", "pass_through_endpoint"),
("pass_through_endpoint", None, "/chunks", "litellm.pass_through", "pass_through_endpoint"),
("aembedding", "retrieval", "/chat", "embeddings", "aembedding"),
("acompletion", "retrieval", "/embeddings", "chat", "acompletion"),
("text_completion", "retrieval", "/chat", "text_completion", "text_completion"),
("allm_passthrough_route", "retrieval", "/embeddings", "chat", "allm_passthrough_route"),
("passthrough_image_generation", "retrieval", "/embeddings", "chat", "passthrough_image_generation"),
],
)
def test_passthrough_operation_v1_span_metrics_and_name(
experimental: bool,
call_type: str,
header: str | None,
path: str,
resolved: str,
legacy: str,
) -> None:
reader: Final = InMemoryMetricReader()
meter: Final = MeterProvider(metric_readers=[reader])
provider: Final = TracerProvider()
otel: Final = OpenTelemetry(
config=OpenTelemetryConfig(
exporter="console",
enable_metrics=True,
semconv_stability_opt_in=({OTELSemconvCategory.GEN_AI_LATEST_EXPERIMENTAL} if experimental else set()),
),
tracer_provider=provider,
meter_provider=meter,
)
kwargs: Final = {
"call_type": call_type,
"model": "test-model",
"optional_params": {},
"litellm_params": {},
"passthrough_operation_name": header,
"passthrough_logging_payload": {"url": f"https://upstream{path}"},
"standard_logging_object": {"call_type": call_type, "model": "test-model", "metadata": {}},
**({} if call_type == "pass_through_endpoint" else {1: "not a string key"}),
}
with provider.get_tracer(__name__).start_as_current_span("test") as span:
otel.set_attributes(span, kwargs, {})
assert span.attributes["gen_ai.operation.name"] == (resolved if experimental else legacy)
assert otel._gen_ai_operation_name(kwargs) == resolved
assert otel._get_span_name(kwargs) == (f"{resolved} test-model" if experimental else "litellm_request")
now: Final = datetime(2026, 1, 1)
otel._record_metrics(kwargs, {}, now, now)
metrics: Final = reader.get_metrics_data()
assert metrics is not None
scopes: Final = chain.from_iterable(resource.scope_metrics for resource in metrics.resource_metrics)
instruments: Final = chain.from_iterable(scope.metrics for scope in scopes)
points: Final = chain.from_iterable(metric.data.data_points for metric in instruments)
operations: Final = tuple(point.attributes["gen_ai.operation.name"] for point in points)
assert operations
assert set(operations) == {resolved}
meter.shutdown()
provider.shutdown()
exporter = InMemorySpanExporter()

View file

@ -1,17 +1,65 @@
import httpx
import pytest
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
from litellm.proxy.pass_through_endpoints.upstream_usage_headers import (
UpstreamReportedUsage,
apply_upstream_reported_usage,
apply_upstream_operation_name,
parse_upstream_reported_usage,
)
from litellm.types.utils import Usage
@pytest.mark.parametrize(
("url", "inferred"),
[
("https://upstream/models/x:generateContent?key=sentinel-secret", "chat"),
("https://user:sentinel-secret@upstream/embeddings?key=sentinel-secret", "embeddings"),
("https://upstream/vectordb/chunks?key=sentinel-secret", "litellm.pass_through"),
],
)
@pytest.mark.parametrize(
("raw", "expected"),
[
("retrieval", "retrieval"),
(" EMBEDDINGS ", "embeddings"),
("vendor.op_1", "vendor.op_1"),
("", None),
(" ", None),
("bad/name", None),
("1chat", None),
("a" * 65, None),
("a" * 64, "a" * 64),
],
)
def test_upstream_operation_reaches_provider_built_payload(
raw: str, expected: str | None, url: str, inferred: str, caplog: pytest.LogCaptureFixture
) -> None:
logging_obj: Final = _logging_obj()
logging_obj.model_call_details["passthrough_logging_payload"] = {"url": url}
apply_upstream_operation_name(logging_obj, httpx.Headers({"x-litellm-operation-name": raw}))
now: Final = datetime(2026, 1, 1)
payload: Final = get_standard_logging_object_payload(
kwargs={"call_type": "pass_through_endpoint", "model": "unknown"},
init_response_obj={"chunks": []},
start_time=now,
end_time=now,
logging_obj=logging_obj,
status="success",
)
assert payload is not None
assert payload["passthrough_operation_name"] == expected
assert payload["passthrough_inferred_operation"] == inferred
assert "passthrough_url" not in payload
assert "sentinel-secret" not in str(payload)
assert ("invalid x-litellm-operation-name" in caplog.text) == (expected is None)
def _headers(**values: str) -> httpx.Headers:
return httpx.Headers(values)