From bb74bb980e7cb66993de328a304b0ec071ff28a8 Mon Sep 17 00:00:00 2001 From: shivam Date: Tue, 27 Oct 2026 23:42:25 +0000 Subject: [PATCH] fix(otel): label pass-through operations from upstream header or route instead of chat Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/opentelemetry.py | 25 +++-- .../opentelemetry_utils/gen_ai_semconv.py | 6 ++ litellm/integrations/otel/model/metadata.py | 2 +- litellm/integrations/otel/model/payloads.py | 5 +- litellm/integrations/otel/model/semconv.py | 30 +++++- litellm/integrations/otel/plumbing/metrics.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 14 +++ .../passthrough_operation.py | 59 +++++++++++ .../pass_through_endpoints.py | 3 + .../upstream_usage_headers.py | 18 ++++ litellm/types/utils.py | 2 + .../otel/test_otel_v2_sources_of_truth.py | 100 +++++++++++++++++- tests/unit/integrations/test_opentelemetry.py | 67 ++++++++++++ .../test_upstream_usage_headers.py | 50 ++++++++- 14 files changed, 368 insertions(+), 15 deletions(-) create mode 100644 litellm/litellm_core_utils/passthrough_operation.py diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0d0e309034a..355abe78cda 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -36,6 +36,10 @@ from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.integrations.otel.routing import routing_decision_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params +from litellm.litellm_core_utils.passthrough_operation import ( + PASSTHROUGH_OPERATION_DATA, + passthrough_header_operation, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.service_tier_utils import ( @@ -1707,9 +1711,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): provider: Final = _provider_label(params.get("custom_llm_provider")) common_attrs = { - "gen_ai.operation.name": ( - self._gen_ai_operation_name(kwargs) if self._gen_ai_semconv_latest_experimental else "chat" - ), + "gen_ai.operation.name": self._gen_ai_operation_name(kwargs), "gen_ai.system": provider, "gen_ai.request.model": kwargs.get("model"), "gen_ai.framework": "litellm", @@ -2687,14 +2689,23 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=safe_dumps(transformed_system_instructions), ) - if self._gen_ai_semconv_latest_experimental: - operation_name = self._gen_ai_operation_name(kwargs) - else: - operation_name = ( + operation_name: Final = ( + self._gen_ai_operation_name(kwargs) + if self._gen_ai_semconv_latest_experimental + else ( + passthrough_header_operation(PASSTHROUGH_OPERATION_DATA.validate_python(kwargs)) + or passthrough_header_operation( + PASSTHROUGH_OPERATION_DATA.validate_python(standard_logging_payload) + ) + or "pass_through_endpoint" + ) + if standard_logging_payload["call_type"] == "pass_through_endpoint" + else ( "chat" if standard_logging_payload.get("call_type") == "completion" else standard_logging_payload.get("call_type") or "chat" ) + ) self.safe_set_attribute( span=span, key=SpanAttributes.GEN_AI_OPERATION_NAME.value, diff --git a/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py b/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py index 035e5c5db10..65525d29d5e 100644 --- a/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py +++ b/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py @@ -35,6 +35,10 @@ from enum import Enum from typing import TYPE_CHECKING, Final, Protocol from litellm.integrations.otel.mappers.utils import drop_none +from litellm.litellm_core_utils.passthrough_operation import ( + PASSTHROUGH_OPERATION_DATA, + resolve_passthrough_operation, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps if TYPE_CHECKING: @@ -151,6 +155,8 @@ class OTELGenAISemconvMixin: ``chat``. """ call_type: Final = kwargs.get("call_type", "") or "" + if call_type == "pass_through_endpoint": + return resolve_passthrough_operation(PASSTHROUGH_OPERATION_DATA.validate_python(kwargs)) match call_type: case s if "embedding" in s: return "embeddings" diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index a69c334a1fb..cd12ec3763e 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -266,7 +266,7 @@ class LLMCallEvent: def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent: raw_payload: Final = kwargs.get("standard_logging_object") payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None - operation: Final = resolve_operation(as_str(kwargs.get("call_type"))) + operation: Final = resolve_operation(as_str(kwargs.get("call_type")), kwargs) model: Final = as_str(kwargs.get("model")) or "" trace: Final = caller_trace_controls(kwargs) return cls( diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 9c62c45e489..b4d9ae4a2e5 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -17,6 +17,7 @@ from litellm.integrations.otel.model.semconv import ( GenAIOperation, GenAIOutputType, MCPMethod, + UpstreamOperation, resolve_operation, resolve_output_type, resolve_provider, @@ -405,7 +406,7 @@ class EmbeddingOutput: @dataclass(frozen=True) class LLMCallSpanData: - operation: GenAIOperation + operation: GenAIOperation | UpstreamOperation provider: str request_model: str response_model: str | None @@ -479,7 +480,7 @@ class LLMCallSpanData: # no prompt/response text. finish_reasons: Final = _finish_reasons(choices_out) call_type: Final = as_str(payload.get("call_type")) - operation: Final = resolve_operation(call_type) + operation: Final = resolve_operation(call_type, payload) embedding_output: Final = ( EmbeddingOutput.from_response(response) if operation is GenAIOperation.EMBEDDINGS else None ) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index b7f955a75ed..6d1ff244775 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -4,11 +4,13 @@ without a semconv equivalent lives under the ``litellm.*`` vendor namespace. """ from collections.abc import Mapping +from dataclasses import dataclass from enum import Enum from types import MappingProxyType -from typing import Final +from typing import Final, overload from litellm._logging import verbose_logger +from litellm.litellm_core_utils.passthrough_operation import resolve_passthrough_operation class GenAIOperation(str, Enum): @@ -34,6 +36,7 @@ class GenAIOperation(str, Enum): LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management" LITELLM_RESPONSES_MANAGEMENT = "litellm.responses_management" LITELLM_MODERATION = "litellm.moderation" + LITELLM_PASS_THROUGH = "litellm.pass_through" class GenAIOutputType(str, Enum): @@ -482,7 +485,22 @@ def resolve_provider(custom_llm_provider: str | None) -> str: return mapped.value if mapped is not None else custom_llm_provider -def resolve_operation(call_type: str | None) -> GenAIOperation: +@dataclass(frozen=True, slots=True) +class UpstreamOperation: + value: str + + +@overload +def resolve_operation(call_type: str | None, data: None = None) -> GenAIOperation: ... + + +@overload +def resolve_operation(call_type: str | None, data: Mapping[str, object]) -> GenAIOperation | UpstreamOperation: ... + + +def resolve_operation( + call_type: str | None, data: Mapping[str, object] | None = None +) -> GenAIOperation | UpstreamOperation: """Map a litellm ``call_type`` to a ``gen_ai.operation.name`` value. An unmapped call type still falls back to ``chat`` so every series keeps an @@ -490,6 +508,14 @@ def resolve_operation(call_type: str | None) -> GenAIOperation: a new call type mislabelled as ``chat`` mixes its latency and cost into everyone's chat charts, which is invisible until someone reads the numbers. """ + if call_type == "pass_through_endpoint": + if data is None: + return GenAIOperation.LITELLM_PASS_THROUGH + passthrough: Final = resolve_passthrough_operation(data) + try: + return GenAIOperation(passthrough) + except ValueError: + return UpstreamOperation(passthrough) if not call_type: return GenAIOperation.CHAT mapped: Final = _OPERATION_BY_CALL_TYPE.get(call_type.lower()) diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 72d51471ba2..c2d1c47d8d6 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -280,7 +280,7 @@ class GenAIMetricRecorder: def _common_attributes(self, kwargs: Mapping[str, Any]) -> dict: params: Final = kwargs.get("litellm_params") or {} common_attrs: Final[dict] = { - GenAI.OPERATION_NAME: resolve_operation(kwargs.get("call_type")).value, + GenAI.OPERATION_NAME: resolve_operation(kwargs.get("call_type"), kwargs).value, **_provider_attributes(params.get("custom_llm_provider")), GenAI.REQUEST_MODEL: kwargs.get("model"), "gen_ai.framework": "litellm", diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 48e8a44b9a4..dd5fa011956 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -120,6 +120,10 @@ from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages_async, ) from litellm.litellm_core_utils.model_param_helper import ModelParamHelper +from litellm.litellm_core_utils.passthrough_operation import ( + infer_passthrough_operation, + normalize_passthrough_operation, +) from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, @@ -7564,6 +7568,16 @@ def get_standard_logging_object_payload( litellm_params=litellm_params, ), call_type=call_type or "", + passthrough_operation_name=( + normalize_passthrough_operation(logging_obj.model_call_details.get("passthrough_operation_name")) + if call_type == "pass_through_endpoint" + else None + ), + passthrough_inferred_operation=( + infer_passthrough_operation(logging_obj.model_call_details) + if call_type == "pass_through_endpoint" + else None + ), cache_hit=cache_hit, stream=stream, status=status, diff --git a/litellm/litellm_core_utils/passthrough_operation.py b/litellm/litellm_core_utils/passthrough_operation.py new file mode 100644 index 00000000000..0f04c46ccf3 --- /dev/null +++ b/litellm/litellm_core_utils/passthrough_operation.py @@ -0,0 +1,59 @@ +import re +from collections.abc import Mapping +from typing import Final +from urllib.parse import urlsplit + +from pydantic import TypeAdapter, ValidationError + +PASSTHROUGH_OPERATION_DATA: Final = TypeAdapter(Mapping[str, object]) + + +def normalize_passthrough_operation(value: object) -> str | None: + if not isinstance(value, str): + return None + normalized: Final = value.strip().lower() + return normalized if re.fullmatch(r"[a-z][a-z0-9_.]{0,63}", normalized) else None + + +def _mapping(value: object) -> Mapping[str, object]: + try: + return PASSTHROUGH_OPERATION_DATA.validate_python(value) + except ValidationError: + return {} + + +def passthrough_header_operation(data: Mapping[str, object]) -> str | None: + return normalize_passthrough_operation(data.get("passthrough_operation_name")) + + +def infer_passthrough_operation(data: Mapping[str, object]) -> str: + inferred: Final = data.get("passthrough_inferred_operation") + if isinstance(inferred, str) and inferred in ("chat", "embeddings", "text_completion", "litellm.pass_through"): + return inferred + url: Final = _mapping(data.get("passthrough_logging_payload")).get("url") + try: + path: Final = urlsplit(url if isinstance(url, str) else "").path.lower() + except ValueError: + return "litellm.pass_through" + if path.endswith( + ( + "/chat/completions", + "/v1/messages", + "/responses", + ":generatecontent", + ":streamgeneratecontent", + "/converse", + "/converse-stream", + "/chat", + ) + ): + return "chat" + if path.endswith(("/embeddings", ":embedcontent", ":batchembedcontents", "/embed")): + return "embeddings" + if path.endswith("/completions"): + return "text_completion" + return "litellm.pass_through" + + +def resolve_passthrough_operation(data: Mapping[str, object]) -> str: + return passthrough_header_operation(data) or infer_passthrough_operation(data) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 97f2c10afb1..afb87cc600c 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -150,6 +150,7 @@ from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging from .upstream_usage_headers import ( UpstreamReportedUsage, + apply_upstream_operation_name, apply_upstream_reported_usage, ) @@ -1539,6 +1540,7 @@ async def pass_through_request( response = await async_client.send(req, stream=stream) + apply_upstream_operation_name(logging_obj, response.headers) upstream_usage = apply_upstream_reported_usage( logging_obj=logging_obj, headers=response.headers, @@ -1627,6 +1629,7 @@ async def pass_through_request( ) verbose_proxy_logger.debug("response.headers= %s", response.headers) + apply_upstream_operation_name(logging_obj, response.headers) upstream_usage = apply_upstream_reported_usage( logging_obj=logging_obj, headers=response.headers, diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index e70d583a027..24901f9aedd 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -15,10 +15,28 @@ import httpx from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.passthrough_operation import normalize_passthrough_operation from litellm.types.utils import Usage UPSTREAM_RESPONSE_COST_HEADER: Final = "x-litellm-response-cost" UPSTREAM_TOTAL_TOKENS_HEADER: Final = "x-litellm-total-tokens" +UPSTREAM_OPERATION_NAME_HEADER: Final = "x-litellm-operation-name" + + +def apply_upstream_operation_name(logging_obj: LiteLLMLoggingObj, headers: httpx.Headers) -> None: + raw: Final = headers.get(UPSTREAM_OPERATION_NAME_HEADER) + if raw is None: + return + operation: Final = normalize_passthrough_operation(raw) + if operation is None: + verbose_proxy_logger.warning( + "pass_through_endpoint: upstream sent invalid %s=%r; ignoring operation name", + UPSTREAM_OPERATION_NAME_HEADER, + raw, + ) + return + logging_obj.model_call_details["passthrough_operation_name"] = operation # rebind-ok: request callback state + # model_call_details key holding what the upstream reported, so later stages of # the success path can tell an upstream-reported cost apart from one LiteLLM diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7e8645fd0dd..b0d8c719b71 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3789,6 +3789,8 @@ class ClassifierAudit(TypedDict, total=False): class StandardLoggingPayload(ClassifierAudit): + passthrough_operation_name: NotRequired[ReadOnly[str | None]] + passthrough_inferred_operation: NotRequired[ReadOnly[str | None]] id: str trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries) session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index 318859cb3ed..daabe37fea1 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -4,7 +4,7 @@ and the typed StandardLoggingPayload adapter. These need no OTel SDK.""" import json import logging import re -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from pathlib import Path from typing import Final @@ -47,6 +47,11 @@ from litellm.integrations.otel.model.spans import ( validate_registry, ) from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls +from litellm.integrations.otel.plumbing.metrics import GenAIMetricRecorder, create_genai_metrics +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from datetime import datetime +from itertools import chain @pytest.fixture(autouse=True) @@ -148,6 +153,99 @@ def test_llm_call_span_name(): assert spans_mod.llm_call_span_name(data) == "chat gpt-4o" +@pytest.mark.parametrize( + ("header", "path", "inferred", "expected"), + [ + ("retrieval", "/embeddings", "embeddings", "retrieval"), + (" EMBEDDINGS ", "/v1/messages", "chat", "embeddings"), + ("vendor.custom", "/responses", "chat", "vendor.custom"), + (None, "/embeddings", "embeddings", "embeddings"), + ("bad/value", "/v1/messages", "chat", "chat"), + (None, "/completions", "text_completion", "text_completion"), + (None, "/unknown", "litellm.pass_through", "litellm.pass_through"), + (None, "", "litellm.pass_through", "litellm.pass_through"), + *[ + (None, path, "chat", "chat") + for path in ( + "/chat/completions", + "/responses", + "/models/x:generateContent", + "/models/x:streamGenerateContent", + "/converse", + "/converse-stream", + "/chat", + ) + ], + *[ + (None, path, "embeddings", "embeddings") + for path in ("/models/x:embedContent", "/models/x:batchEmbedContents", "/embed") + ], + ], +) +def test_passthrough_operation_agrees_in_v2_span_name_attributes_and_metrics( + header: str | None, path: str, inferred: str, expected: str +) -> None: + payload: Final = _sample_payload( + call_type="pass_through_endpoint", + passthrough_operation_name=header, + passthrough_inferred_operation=inferred, + ) + kwargs: Final = { + "call_type": "pass_through_endpoint", + "model": "gpt-4o", + "passthrough_operation_name": header, + "passthrough_logging_payload": {"url": f"https://upstream{path}"}, + "standard_logging_object": payload, + } + data: Final = LLMCallSpanData.from_standard_logging_payload(payload) + assert GenAIMapper().map(data)[GenAI.OPERATION_NAME] == expected + assert spans_mod.llm_call_span_name(data) == f"{expected} gpt-4o" + assert LLMCallEvent.from_dict(kwargs).provisional_span_name == f"{expected} gpt-4o" + reader: Final = InMemoryMetricReader() + provider: Final = MeterProvider(metric_readers=[reader]) + recorder: Final = GenAIMetricRecorder(create_genai_metrics(provider.get_meter(__name__))) + now: Final = datetime(2026, 1, 1) + recorder.record(kwargs, None, now, now) + metrics: Final = reader.get_metrics_data() + assert metrics is not None + scopes: Final = chain.from_iterable(resource.scope_metrics for resource in metrics.resource_metrics) + instruments: Final = chain.from_iterable(scope.metrics for scope in scopes) + points: Final = chain.from_iterable(metric.data.data_points for metric in instruments) + operations: Final = tuple(point.attributes[GenAI.OPERATION_NAME] for point in points) + assert operations + assert set(operations) == {expected} + provider.shutdown() + + +@pytest.mark.parametrize( + "call_type", + ["acompletion", "aembedding", "text_completion", "allm_passthrough_route", "passthrough_image_generation"], +) +def test_upstream_operation_does_not_override_other_v2_call_types(call_type: str) -> None: + baseline: Final = LLMCallSpanData.from_standard_logging_payload(_sample_payload(call_type=call_type)) + overridden: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload( + call_type=call_type, passthrough_operation_name="retrieval", passthrough_inferred_operation="embeddings" + ) + ) + assert GenAIMapper().map(overridden) == GenAIMapper().map(baseline) + assert spans_mod.llm_call_span_name(overridden) == spans_mod.llm_call_span_name(baseline) + + +def test_non_passthrough_operation_does_not_read_upstream_data() -> None: + class UnreadableOperationData(Mapping[str, object]): + def __getitem__(self, key: str) -> object: + raise AssertionError("non-pass-through calls must not read upstream data") + + def __iter__(self) -> Iterator[str]: + raise AssertionError("non-pass-through calls must not copy upstream data") + + def __len__(self) -> int: + raise AssertionError("non-pass-through calls must not inspect upstream data") + + assert resolve_operation("acompletion", UnreadableOperationData()) is GenAIOperation.CHAT + + # --- semconv (source of truth #1) ------------------------------------------- # diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index f908c685382..f1b30113429 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -13,6 +13,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from types import MappingProxyType from typing import Final +from itertools import chain from unittest.mock import MagicMock, patch import pytest @@ -59,6 +60,72 @@ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import ChatCompletionMessageToolCall, Function from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +@pytest.mark.parametrize("experimental", [False, True]) +@pytest.mark.parametrize( + ("call_type", "header", "path", "resolved", "legacy"), + [ + ("pass_through_endpoint", "retrieval", "/embeddings", "retrieval", "retrieval"), + ("pass_through_endpoint", " EMBEDDINGS ", "/v1/messages", "embeddings", "embeddings"), + ("pass_through_endpoint", None, "/embeddings", "embeddings", "pass_through_endpoint"), + ("pass_through_endpoint", "bad/value", "/v1/messages", "chat", "pass_through_endpoint"), + ("pass_through_endpoint", None, "/completions", "text_completion", "pass_through_endpoint"), + ("pass_through_endpoint", None, "/chunks", "litellm.pass_through", "pass_through_endpoint"), + ("aembedding", "retrieval", "/chat", "embeddings", "aembedding"), + ("acompletion", "retrieval", "/embeddings", "chat", "acompletion"), + ("text_completion", "retrieval", "/chat", "text_completion", "text_completion"), + ("allm_passthrough_route", "retrieval", "/embeddings", "chat", "allm_passthrough_route"), + ("passthrough_image_generation", "retrieval", "/embeddings", "chat", "passthrough_image_generation"), + ], +) +def test_passthrough_operation_v1_span_metrics_and_name( + experimental: bool, + call_type: str, + header: str | None, + path: str, + resolved: str, + legacy: str, +) -> None: + reader: Final = InMemoryMetricReader() + meter: Final = MeterProvider(metric_readers=[reader]) + provider: Final = TracerProvider() + otel: Final = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", + enable_metrics=True, + semconv_stability_opt_in=({OTELSemconvCategory.GEN_AI_LATEST_EXPERIMENTAL} if experimental else set()), + ), + tracer_provider=provider, + meter_provider=meter, + ) + kwargs: Final = { + "call_type": call_type, + "model": "test-model", + "optional_params": {}, + "litellm_params": {}, + "passthrough_operation_name": header, + "passthrough_logging_payload": {"url": f"https://upstream{path}"}, + "standard_logging_object": {"call_type": call_type, "model": "test-model", "metadata": {}}, + **({} if call_type == "pass_through_endpoint" else {1: "not a string key"}), + } + with provider.get_tracer(__name__).start_as_current_span("test") as span: + otel.set_attributes(span, kwargs, {}) + assert span.attributes["gen_ai.operation.name"] == (resolved if experimental else legacy) + assert otel._gen_ai_operation_name(kwargs) == resolved + assert otel._get_span_name(kwargs) == (f"{resolved} test-model" if experimental else "litellm_request") + now: Final = datetime(2026, 1, 1) + otel._record_metrics(kwargs, {}, now, now) + metrics: Final = reader.get_metrics_data() + assert metrics is not None + scopes: Final = chain.from_iterable(resource.scope_metrics for resource in metrics.resource_metrics) + instruments: Final = chain.from_iterable(scope.metrics for scope in scopes) + points: Final = chain.from_iterable(metric.data.data_points for metric in instruments) + operations: Final = tuple(point.attributes["gen_ai.operation.name"] for point in points) + assert operations + assert set(operations) == {resolved} + meter.shutdown() + provider.shutdown() + + exporter = InMemorySpanExporter() diff --git a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py index 701c583a4ab..ae62d590850 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py +++ b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py @@ -1,17 +1,65 @@ - import httpx import pytest +from datetime import datetime +from typing import Final from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload from litellm.proxy.pass_through_endpoints.upstream_usage_headers import ( UpstreamReportedUsage, apply_upstream_reported_usage, + apply_upstream_operation_name, parse_upstream_reported_usage, ) from litellm.types.utils import Usage +@pytest.mark.parametrize( + ("url", "inferred"), + [ + ("https://upstream/models/x:generateContent?key=sentinel-secret", "chat"), + ("https://user:sentinel-secret@upstream/embeddings?key=sentinel-secret", "embeddings"), + ("https://upstream/vectordb/chunks?key=sentinel-secret", "litellm.pass_through"), + ], +) +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("retrieval", "retrieval"), + (" EMBEDDINGS ", "embeddings"), + ("vendor.op_1", "vendor.op_1"), + ("", None), + (" ", None), + ("bad/name", None), + ("1chat", None), + ("a" * 65, None), + ("a" * 64, "a" * 64), + ], +) +def test_upstream_operation_reaches_provider_built_payload( + raw: str, expected: str | None, url: str, inferred: str, caplog: pytest.LogCaptureFixture +) -> None: + logging_obj: Final = _logging_obj() + logging_obj.model_call_details["passthrough_logging_payload"] = {"url": url} + apply_upstream_operation_name(logging_obj, httpx.Headers({"x-litellm-operation-name": raw})) + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={"call_type": "pass_through_endpoint", "model": "unknown"}, + init_response_obj={"chunks": []}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + assert payload is not None + assert payload["passthrough_operation_name"] == expected + assert payload["passthrough_inferred_operation"] == inferred + assert "passthrough_url" not in payload + assert "sentinel-secret" not in str(payload) + assert ("invalid x-litellm-operation-name" in caplog.text) == (expected is None) + + def _headers(**values: str) -> httpx.Headers: return httpx.Headers(values)