mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(otel): label pass-through operations from upstream header or route instead of chat
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
cedae29cc7
commit
bb74bb980e
14 changed files with 368 additions and 15 deletions
|
|
@ -36,6 +36,10 @@ from litellm.integrations.otel.model.semconv import LiteLLM, Metric
|
|||
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
|
||||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
|
||||
from litellm.litellm_core_utils.passthrough_operation import (
|
||||
PASSTHROUGH_OPERATION_DATA,
|
||||
passthrough_header_operation,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
from litellm.litellm_core_utils.service_tier_utils import (
|
||||
|
|
@ -1707,9 +1711,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
provider: Final = _provider_label(params.get("custom_llm_provider"))
|
||||
|
||||
common_attrs = {
|
||||
"gen_ai.operation.name": (
|
||||
self._gen_ai_operation_name(kwargs) if self._gen_ai_semconv_latest_experimental else "chat"
|
||||
),
|
||||
"gen_ai.operation.name": self._gen_ai_operation_name(kwargs),
|
||||
"gen_ai.system": provider,
|
||||
"gen_ai.request.model": kwargs.get("model"),
|
||||
"gen_ai.framework": "litellm",
|
||||
|
|
@ -2687,14 +2689,23 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
value=safe_dumps(transformed_system_instructions),
|
||||
)
|
||||
|
||||
if self._gen_ai_semconv_latest_experimental:
|
||||
operation_name = self._gen_ai_operation_name(kwargs)
|
||||
else:
|
||||
operation_name = (
|
||||
operation_name: Final = (
|
||||
self._gen_ai_operation_name(kwargs)
|
||||
if self._gen_ai_semconv_latest_experimental
|
||||
else (
|
||||
passthrough_header_operation(PASSTHROUGH_OPERATION_DATA.validate_python(kwargs))
|
||||
or passthrough_header_operation(
|
||||
PASSTHROUGH_OPERATION_DATA.validate_python(standard_logging_payload)
|
||||
)
|
||||
or "pass_through_endpoint"
|
||||
)
|
||||
if standard_logging_payload["call_type"] == "pass_through_endpoint"
|
||||
else (
|
||||
"chat"
|
||||
if standard_logging_payload.get("call_type") == "completion"
|
||||
else standard_logging_payload.get("call_type") or "chat"
|
||||
)
|
||||
)
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=SpanAttributes.GEN_AI_OPERATION_NAME.value,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,10 @@ from enum import Enum
|
|||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from litellm.integrations.otel.mappers.utils import drop_none
|
||||
from litellm.litellm_core_utils.passthrough_operation import (
|
||||
PASSTHROUGH_OPERATION_DATA,
|
||||
resolve_passthrough_operation,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -151,6 +155,8 @@ class OTELGenAISemconvMixin:
|
|||
``chat``.
|
||||
"""
|
||||
call_type: Final = kwargs.get("call_type", "") or ""
|
||||
if call_type == "pass_through_endpoint":
|
||||
return resolve_passthrough_operation(PASSTHROUGH_OPERATION_DATA.validate_python(kwargs))
|
||||
match call_type:
|
||||
case s if "embedding" in s:
|
||||
return "embeddings"
|
||||
|
|
|
|||
|
|
@ -266,7 +266,7 @@ class LLMCallEvent:
|
|||
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
|
||||
raw_payload: Final = kwargs.get("standard_logging_object")
|
||||
payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None
|
||||
operation: Final = resolve_operation(as_str(kwargs.get("call_type")))
|
||||
operation: Final = resolve_operation(as_str(kwargs.get("call_type")), kwargs)
|
||||
model: Final = as_str(kwargs.get("model")) or ""
|
||||
trace: Final = caller_trace_controls(kwargs)
|
||||
return cls(
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.integrations.otel.model.semconv import (
|
|||
GenAIOperation,
|
||||
GenAIOutputType,
|
||||
MCPMethod,
|
||||
UpstreamOperation,
|
||||
resolve_operation,
|
||||
resolve_output_type,
|
||||
resolve_provider,
|
||||
|
|
@ -405,7 +406,7 @@ class EmbeddingOutput:
|
|||
|
||||
@dataclass(frozen=True)
|
||||
class LLMCallSpanData:
|
||||
operation: GenAIOperation
|
||||
operation: GenAIOperation | UpstreamOperation
|
||||
provider: str
|
||||
request_model: str
|
||||
response_model: str | None
|
||||
|
|
@ -479,7 +480,7 @@ class LLMCallSpanData:
|
|||
# no prompt/response text.
|
||||
finish_reasons: Final = _finish_reasons(choices_out)
|
||||
call_type: Final = as_str(payload.get("call_type"))
|
||||
operation: Final = resolve_operation(call_type)
|
||||
operation: Final = resolve_operation(call_type, payload)
|
||||
embedding_output: Final = (
|
||||
EmbeddingOutput.from_response(response) if operation is GenAIOperation.EMBEDDINGS else None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,11 +4,13 @@ without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
|
|||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, overload
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.passthrough_operation import resolve_passthrough_operation
|
||||
|
||||
|
||||
class GenAIOperation(str, Enum):
|
||||
|
|
@ -34,6 +36,7 @@ class GenAIOperation(str, Enum):
|
|||
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
|
||||
LITELLM_RESPONSES_MANAGEMENT = "litellm.responses_management"
|
||||
LITELLM_MODERATION = "litellm.moderation"
|
||||
LITELLM_PASS_THROUGH = "litellm.pass_through"
|
||||
|
||||
|
||||
class GenAIOutputType(str, Enum):
|
||||
|
|
@ -482,7 +485,22 @@ def resolve_provider(custom_llm_provider: str | None) -> str:
|
|||
return mapped.value if mapped is not None else custom_llm_provider
|
||||
|
||||
|
||||
def resolve_operation(call_type: str | None) -> GenAIOperation:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UpstreamOperation:
|
||||
value: str
|
||||
|
||||
|
||||
@overload
|
||||
def resolve_operation(call_type: str | None, data: None = None) -> GenAIOperation: ...
|
||||
|
||||
|
||||
@overload
|
||||
def resolve_operation(call_type: str | None, data: Mapping[str, object]) -> GenAIOperation | UpstreamOperation: ...
|
||||
|
||||
|
||||
def resolve_operation(
|
||||
call_type: str | None, data: Mapping[str, object] | None = None
|
||||
) -> GenAIOperation | UpstreamOperation:
|
||||
"""Map a litellm ``call_type`` to a ``gen_ai.operation.name`` value.
|
||||
|
||||
An unmapped call type still falls back to ``chat`` so every series keeps an
|
||||
|
|
@ -490,6 +508,14 @@ def resolve_operation(call_type: str | None) -> GenAIOperation:
|
|||
a new call type mislabelled as ``chat`` mixes its latency and cost into
|
||||
everyone's chat charts, which is invisible until someone reads the numbers.
|
||||
"""
|
||||
if call_type == "pass_through_endpoint":
|
||||
if data is None:
|
||||
return GenAIOperation.LITELLM_PASS_THROUGH
|
||||
passthrough: Final = resolve_passthrough_operation(data)
|
||||
try:
|
||||
return GenAIOperation(passthrough)
|
||||
except ValueError:
|
||||
return UpstreamOperation(passthrough)
|
||||
if not call_type:
|
||||
return GenAIOperation.CHAT
|
||||
mapped: Final = _OPERATION_BY_CALL_TYPE.get(call_type.lower())
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ class GenAIMetricRecorder:
|
|||
def _common_attributes(self, kwargs: Mapping[str, Any]) -> dict:
|
||||
params: Final = kwargs.get("litellm_params") or {}
|
||||
common_attrs: Final[dict] = {
|
||||
GenAI.OPERATION_NAME: resolve_operation(kwargs.get("call_type")).value,
|
||||
GenAI.OPERATION_NAME: resolve_operation(kwargs.get("call_type"), kwargs).value,
|
||||
**_provider_attributes(params.get("custom_llm_provider")),
|
||||
GenAI.REQUEST_MODEL: kwargs.get("model"),
|
||||
"gen_ai.framework": "litellm",
|
||||
|
|
|
|||
|
|
@ -120,6 +120,10 @@ from litellm.litellm_core_utils.logging_utils import (
|
|||
truncate_base64_in_messages_async,
|
||||
)
|
||||
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
|
||||
from litellm.litellm_core_utils.passthrough_operation import (
|
||||
infer_passthrough_operation,
|
||||
normalize_passthrough_operation,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
redact_message_input_output_from_custom_logger,
|
||||
|
|
@ -7564,6 +7568,16 @@ def get_standard_logging_object_payload(
|
|||
litellm_params=litellm_params,
|
||||
),
|
||||
call_type=call_type or "",
|
||||
passthrough_operation_name=(
|
||||
normalize_passthrough_operation(logging_obj.model_call_details.get("passthrough_operation_name"))
|
||||
if call_type == "pass_through_endpoint"
|
||||
else None
|
||||
),
|
||||
passthrough_inferred_operation=(
|
||||
infer_passthrough_operation(logging_obj.model_call_details)
|
||||
if call_type == "pass_through_endpoint"
|
||||
else None
|
||||
),
|
||||
cache_hit=cache_hit,
|
||||
stream=stream,
|
||||
status=status,
|
||||
|
|
|
|||
59
litellm/litellm_core_utils/passthrough_operation.py
Normal file
59
litellm/litellm_core_utils/passthrough_operation.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
PASSTHROUGH_OPERATION_DATA: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def normalize_passthrough_operation(value: object) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
normalized: Final = value.strip().lower()
|
||||
return normalized if re.fullmatch(r"[a-z][a-z0-9_.]{0,63}", normalized) else None
|
||||
|
||||
|
||||
def _mapping(value: object) -> Mapping[str, object]:
|
||||
try:
|
||||
return PASSTHROUGH_OPERATION_DATA.validate_python(value)
|
||||
except ValidationError:
|
||||
return {}
|
||||
|
||||
|
||||
def passthrough_header_operation(data: Mapping[str, object]) -> str | None:
|
||||
return normalize_passthrough_operation(data.get("passthrough_operation_name"))
|
||||
|
||||
|
||||
def infer_passthrough_operation(data: Mapping[str, object]) -> str:
|
||||
inferred: Final = data.get("passthrough_inferred_operation")
|
||||
if isinstance(inferred, str) and inferred in ("chat", "embeddings", "text_completion", "litellm.pass_through"):
|
||||
return inferred
|
||||
url: Final = _mapping(data.get("passthrough_logging_payload")).get("url")
|
||||
try:
|
||||
path: Final = urlsplit(url if isinstance(url, str) else "").path.lower()
|
||||
except ValueError:
|
||||
return "litellm.pass_through"
|
||||
if path.endswith(
|
||||
(
|
||||
"/chat/completions",
|
||||
"/v1/messages",
|
||||
"/responses",
|
||||
":generatecontent",
|
||||
":streamgeneratecontent",
|
||||
"/converse",
|
||||
"/converse-stream",
|
||||
"/chat",
|
||||
)
|
||||
):
|
||||
return "chat"
|
||||
if path.endswith(("/embeddings", ":embedcontent", ":batchembedcontents", "/embed")):
|
||||
return "embeddings"
|
||||
if path.endswith("/completions"):
|
||||
return "text_completion"
|
||||
return "litellm.pass_through"
|
||||
|
||||
|
||||
def resolve_passthrough_operation(data: Mapping[str, object]) -> str:
|
||||
return passthrough_header_operation(data) or infer_passthrough_operation(data)
|
||||
|
|
@ -150,6 +150,7 @@ from .streaming_handler import PassThroughStreamingHandler
|
|||
from .success_handler import PassThroughEndpointLogging
|
||||
from .upstream_usage_headers import (
|
||||
UpstreamReportedUsage,
|
||||
apply_upstream_operation_name,
|
||||
apply_upstream_reported_usage,
|
||||
)
|
||||
|
||||
|
|
@ -1539,6 +1540,7 @@ async def pass_through_request(
|
|||
|
||||
response = await async_client.send(req, stream=stream)
|
||||
|
||||
apply_upstream_operation_name(logging_obj, response.headers)
|
||||
upstream_usage = apply_upstream_reported_usage(
|
||||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
|
|
@ -1627,6 +1629,7 @@ async def pass_through_request(
|
|||
)
|
||||
verbose_proxy_logger.debug("response.headers= %s", response.headers)
|
||||
|
||||
apply_upstream_operation_name(logging_obj, response.headers)
|
||||
upstream_usage = apply_upstream_reported_usage(
|
||||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
|
|
|
|||
|
|
@ -15,10 +15,28 @@ import httpx
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.passthrough_operation import normalize_passthrough_operation
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
UPSTREAM_RESPONSE_COST_HEADER: Final = "x-litellm-response-cost"
|
||||
UPSTREAM_TOTAL_TOKENS_HEADER: Final = "x-litellm-total-tokens"
|
||||
UPSTREAM_OPERATION_NAME_HEADER: Final = "x-litellm-operation-name"
|
||||
|
||||
|
||||
def apply_upstream_operation_name(logging_obj: LiteLLMLoggingObj, headers: httpx.Headers) -> None:
|
||||
raw: Final = headers.get(UPSTREAM_OPERATION_NAME_HEADER)
|
||||
if raw is None:
|
||||
return
|
||||
operation: Final = normalize_passthrough_operation(raw)
|
||||
if operation is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"pass_through_endpoint: upstream sent invalid %s=%r; ignoring operation name",
|
||||
UPSTREAM_OPERATION_NAME_HEADER,
|
||||
raw,
|
||||
)
|
||||
return
|
||||
logging_obj.model_call_details["passthrough_operation_name"] = operation # rebind-ok: request callback state
|
||||
|
||||
|
||||
# model_call_details key holding what the upstream reported, so later stages of
|
||||
# the success path can tell an upstream-reported cost apart from one LiteLLM
|
||||
|
|
|
|||
|
|
@ -3789,6 +3789,8 @@ class ClassifierAudit(TypedDict, total=False):
|
|||
|
||||
|
||||
class StandardLoggingPayload(ClassifierAudit):
|
||||
passthrough_operation_name: NotRequired[ReadOnly[str | None]]
|
||||
passthrough_inferred_operation: NotRequired[ReadOnly[str | None]]
|
||||
id: str
|
||||
trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
|
||||
session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ and the typed StandardLoggingPayload adapter. These need no OTel SDK."""
|
|||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterator, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -47,6 +47,11 @@ from litellm.integrations.otel.model.spans import (
|
|||
validate_registry,
|
||||
)
|
||||
from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls
|
||||
from litellm.integrations.otel.plumbing.metrics import GenAIMetricRecorder, create_genai_metrics
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -148,6 +153,99 @@ def test_llm_call_span_name():
|
|||
assert spans_mod.llm_call_span_name(data) == "chat gpt-4o"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("header", "path", "inferred", "expected"),
|
||||
[
|
||||
("retrieval", "/embeddings", "embeddings", "retrieval"),
|
||||
(" EMBEDDINGS ", "/v1/messages", "chat", "embeddings"),
|
||||
("vendor.custom", "/responses", "chat", "vendor.custom"),
|
||||
(None, "/embeddings", "embeddings", "embeddings"),
|
||||
("bad/value", "/v1/messages", "chat", "chat"),
|
||||
(None, "/completions", "text_completion", "text_completion"),
|
||||
(None, "/unknown", "litellm.pass_through", "litellm.pass_through"),
|
||||
(None, "", "litellm.pass_through", "litellm.pass_through"),
|
||||
*[
|
||||
(None, path, "chat", "chat")
|
||||
for path in (
|
||||
"/chat/completions",
|
||||
"/responses",
|
||||
"/models/x:generateContent",
|
||||
"/models/x:streamGenerateContent",
|
||||
"/converse",
|
||||
"/converse-stream",
|
||||
"/chat",
|
||||
)
|
||||
],
|
||||
*[
|
||||
(None, path, "embeddings", "embeddings")
|
||||
for path in ("/models/x:embedContent", "/models/x:batchEmbedContents", "/embed")
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_passthrough_operation_agrees_in_v2_span_name_attributes_and_metrics(
|
||||
header: str | None, path: str, inferred: str, expected: str
|
||||
) -> None:
|
||||
payload: Final = _sample_payload(
|
||||
call_type="pass_through_endpoint",
|
||||
passthrough_operation_name=header,
|
||||
passthrough_inferred_operation=inferred,
|
||||
)
|
||||
kwargs: Final = {
|
||||
"call_type": "pass_through_endpoint",
|
||||
"model": "gpt-4o",
|
||||
"passthrough_operation_name": header,
|
||||
"passthrough_logging_payload": {"url": f"https://upstream{path}"},
|
||||
"standard_logging_object": payload,
|
||||
}
|
||||
data: Final = LLMCallSpanData.from_standard_logging_payload(payload)
|
||||
assert GenAIMapper().map(data)[GenAI.OPERATION_NAME] == expected
|
||||
assert spans_mod.llm_call_span_name(data) == f"{expected} gpt-4o"
|
||||
assert LLMCallEvent.from_dict(kwargs).provisional_span_name == f"{expected} gpt-4o"
|
||||
reader: Final = InMemoryMetricReader()
|
||||
provider: Final = MeterProvider(metric_readers=[reader])
|
||||
recorder: Final = GenAIMetricRecorder(create_genai_metrics(provider.get_meter(__name__)))
|
||||
now: Final = datetime(2026, 1, 1)
|
||||
recorder.record(kwargs, None, now, now)
|
||||
metrics: Final = reader.get_metrics_data()
|
||||
assert metrics is not None
|
||||
scopes: Final = chain.from_iterable(resource.scope_metrics for resource in metrics.resource_metrics)
|
||||
instruments: Final = chain.from_iterable(scope.metrics for scope in scopes)
|
||||
points: Final = chain.from_iterable(metric.data.data_points for metric in instruments)
|
||||
operations: Final = tuple(point.attributes[GenAI.OPERATION_NAME] for point in points)
|
||||
assert operations
|
||||
assert set(operations) == {expected}
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call_type",
|
||||
["acompletion", "aembedding", "text_completion", "allm_passthrough_route", "passthrough_image_generation"],
|
||||
)
|
||||
def test_upstream_operation_does_not_override_other_v2_call_types(call_type: str) -> None:
|
||||
baseline: Final = LLMCallSpanData.from_standard_logging_payload(_sample_payload(call_type=call_type))
|
||||
overridden: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
_sample_payload(
|
||||
call_type=call_type, passthrough_operation_name="retrieval", passthrough_inferred_operation="embeddings"
|
||||
)
|
||||
)
|
||||
assert GenAIMapper().map(overridden) == GenAIMapper().map(baseline)
|
||||
assert spans_mod.llm_call_span_name(overridden) == spans_mod.llm_call_span_name(baseline)
|
||||
|
||||
|
||||
def test_non_passthrough_operation_does_not_read_upstream_data() -> None:
|
||||
class UnreadableOperationData(Mapping[str, object]):
|
||||
def __getitem__(self, key: str) -> object:
|
||||
raise AssertionError("non-pass-through calls must not read upstream data")
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
raise AssertionError("non-pass-through calls must not copy upstream data")
|
||||
|
||||
def __len__(self) -> int:
|
||||
raise AssertionError("non-pass-through calls must not inspect upstream data")
|
||||
|
||||
assert resolve_operation("acompletion", UnreadableOperationData()) is GenAIOperation.CHAT
|
||||
|
||||
|
||||
# --- semconv (source of truth #1) ------------------------------------------- #
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from datetime import datetime, timedelta, timezone
|
|||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from itertools import chain
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -59,6 +60,72 @@ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
|||
from litellm.types.utils import ChatCompletionMessageToolCall, Function
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
||||
@pytest.mark.parametrize("experimental", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "header", "path", "resolved", "legacy"),
|
||||
[
|
||||
("pass_through_endpoint", "retrieval", "/embeddings", "retrieval", "retrieval"),
|
||||
("pass_through_endpoint", " EMBEDDINGS ", "/v1/messages", "embeddings", "embeddings"),
|
||||
("pass_through_endpoint", None, "/embeddings", "embeddings", "pass_through_endpoint"),
|
||||
("pass_through_endpoint", "bad/value", "/v1/messages", "chat", "pass_through_endpoint"),
|
||||
("pass_through_endpoint", None, "/completions", "text_completion", "pass_through_endpoint"),
|
||||
("pass_through_endpoint", None, "/chunks", "litellm.pass_through", "pass_through_endpoint"),
|
||||
("aembedding", "retrieval", "/chat", "embeddings", "aembedding"),
|
||||
("acompletion", "retrieval", "/embeddings", "chat", "acompletion"),
|
||||
("text_completion", "retrieval", "/chat", "text_completion", "text_completion"),
|
||||
("allm_passthrough_route", "retrieval", "/embeddings", "chat", "allm_passthrough_route"),
|
||||
("passthrough_image_generation", "retrieval", "/embeddings", "chat", "passthrough_image_generation"),
|
||||
],
|
||||
)
|
||||
def test_passthrough_operation_v1_span_metrics_and_name(
|
||||
experimental: bool,
|
||||
call_type: str,
|
||||
header: str | None,
|
||||
path: str,
|
||||
resolved: str,
|
||||
legacy: str,
|
||||
) -> None:
|
||||
reader: Final = InMemoryMetricReader()
|
||||
meter: Final = MeterProvider(metric_readers=[reader])
|
||||
provider: Final = TracerProvider()
|
||||
otel: Final = OpenTelemetry(
|
||||
config=OpenTelemetryConfig(
|
||||
exporter="console",
|
||||
enable_metrics=True,
|
||||
semconv_stability_opt_in=({OTELSemconvCategory.GEN_AI_LATEST_EXPERIMENTAL} if experimental else set()),
|
||||
),
|
||||
tracer_provider=provider,
|
||||
meter_provider=meter,
|
||||
)
|
||||
kwargs: Final = {
|
||||
"call_type": call_type,
|
||||
"model": "test-model",
|
||||
"optional_params": {},
|
||||
"litellm_params": {},
|
||||
"passthrough_operation_name": header,
|
||||
"passthrough_logging_payload": {"url": f"https://upstream{path}"},
|
||||
"standard_logging_object": {"call_type": call_type, "model": "test-model", "metadata": {}},
|
||||
**({} if call_type == "pass_through_endpoint" else {1: "not a string key"}),
|
||||
}
|
||||
with provider.get_tracer(__name__).start_as_current_span("test") as span:
|
||||
otel.set_attributes(span, kwargs, {})
|
||||
assert span.attributes["gen_ai.operation.name"] == (resolved if experimental else legacy)
|
||||
assert otel._gen_ai_operation_name(kwargs) == resolved
|
||||
assert otel._get_span_name(kwargs) == (f"{resolved} test-model" if experimental else "litellm_request")
|
||||
now: Final = datetime(2026, 1, 1)
|
||||
otel._record_metrics(kwargs, {}, now, now)
|
||||
metrics: Final = reader.get_metrics_data()
|
||||
assert metrics is not None
|
||||
scopes: Final = chain.from_iterable(resource.scope_metrics for resource in metrics.resource_metrics)
|
||||
instruments: Final = chain.from_iterable(scope.metrics for scope in scopes)
|
||||
points: Final = chain.from_iterable(metric.data.data_points for metric in instruments)
|
||||
operations: Final = tuple(point.attributes["gen_ai.operation.name"] for point in points)
|
||||
assert operations
|
||||
assert set(operations) == {resolved}
|
||||
meter.shutdown()
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
exporter = InMemorySpanExporter()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,65 @@
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
|
||||
from litellm.proxy.pass_through_endpoints.upstream_usage_headers import (
|
||||
UpstreamReportedUsage,
|
||||
apply_upstream_reported_usage,
|
||||
apply_upstream_operation_name,
|
||||
parse_upstream_reported_usage,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "inferred"),
|
||||
[
|
||||
("https://upstream/models/x:generateContent?key=sentinel-secret", "chat"),
|
||||
("https://user:sentinel-secret@upstream/embeddings?key=sentinel-secret", "embeddings"),
|
||||
("https://upstream/vectordb/chunks?key=sentinel-secret", "litellm.pass_through"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("retrieval", "retrieval"),
|
||||
(" EMBEDDINGS ", "embeddings"),
|
||||
("vendor.op_1", "vendor.op_1"),
|
||||
("", None),
|
||||
(" ", None),
|
||||
("bad/name", None),
|
||||
("1chat", None),
|
||||
("a" * 65, None),
|
||||
("a" * 64, "a" * 64),
|
||||
],
|
||||
)
|
||||
def test_upstream_operation_reaches_provider_built_payload(
|
||||
raw: str, expected: str | None, url: str, inferred: str, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
logging_obj: Final = _logging_obj()
|
||||
logging_obj.model_call_details["passthrough_logging_payload"] = {"url": url}
|
||||
apply_upstream_operation_name(logging_obj, httpx.Headers({"x-litellm-operation-name": raw}))
|
||||
now: Final = datetime(2026, 1, 1)
|
||||
payload: Final = get_standard_logging_object_payload(
|
||||
kwargs={"call_type": "pass_through_endpoint", "model": "unknown"},
|
||||
init_response_obj={"chunks": []},
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
logging_obj=logging_obj,
|
||||
status="success",
|
||||
)
|
||||
assert payload is not None
|
||||
assert payload["passthrough_operation_name"] == expected
|
||||
assert payload["passthrough_inferred_operation"] == inferred
|
||||
assert "passthrough_url" not in payload
|
||||
assert "sentinel-secret" not in str(payload)
|
||||
assert ("invalid x-litellm-operation-name" in caplog.text) == (expected is None)
|
||||
|
||||
|
||||
def _headers(**values: str) -> httpx.Headers:
|
||||
return httpx.Headers(values)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue