mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(otel): support GenAI client suppression and context attribute propagation
This commit is contained in:
parent
03c26c6e07
commit
c0d506b557
6 changed files with 645 additions and 26 deletions
|
|
@ -228,6 +228,25 @@ class SpanEmitter:
|
|||
|
||||
# -- the engine ---------------------------------------------------------- #
|
||||
|
||||
def _map_data(self, data: SpanData) -> tuple[dict[str, AttrValue], Mapping[str, AttrValue]]:
|
||||
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
for_langfuse: Final = (
|
||||
dict(
|
||||
sorted(
|
||||
self._langfuse.map(data).items(),
|
||||
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
)
|
||||
)
|
||||
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
else _NO_ATTRIBUTES
|
||||
)
|
||||
return {**configured, **for_langfuse}, for_langfuse
|
||||
|
||||
def map_attributes(self, data: SpanData) -> dict[str, AttrValue]:
|
||||
"""Run the configured mapper chain on ``data`` without creating a span."""
|
||||
mapped, _ = self._map_data(data)
|
||||
return mapped
|
||||
|
||||
def emit(
|
||||
self,
|
||||
role: SpanRole,
|
||||
|
|
@ -299,18 +318,8 @@ class SpanEmitter:
|
|||
)
|
||||
else None
|
||||
)
|
||||
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
for_langfuse: Final = (
|
||||
dict(
|
||||
sorted(
|
||||
self._langfuse.map(data).items(),
|
||||
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
)
|
||||
)
|
||||
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
else _NO_ATTRIBUTES
|
||||
)
|
||||
mapped: Final = MappingProxyType({**configured, **for_langfuse})
|
||||
raw_mapped, for_langfuse = self._map_data(data)
|
||||
mapped: Final = MappingProxyType(raw_mapped)
|
||||
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
|
||||
reserved: Final = len(stamped_later.keys() - mapped.keys())
|
||||
overwrites: Final = carried_keys(span) & mapped.keys()
|
||||
|
|
|
|||
127
litellm/integrations/otel/genai_context.py
Normal file
127
litellm/integrations/otel/genai_context.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""OpenTelemetry GenAI client context suppression & attribute propagation.
|
||||
|
||||
Integrates ``OpenTelemetryV2`` with ``opentelemetry-util-genai``'s context-sharing
|
||||
contract (see ``open-telemetry/semantic-conventions-genai#682`` and
|
||||
``open-telemetry/opentelemetry-python-genai#859``):
|
||||
|
||||
When LiteLLM is called inside an outer GenAI instrumentation that attached
|
||||
``InferenceData``, ``EmbeddingData``, ``RetrievalData``, or
|
||||
``FetchResponseData`` under ``CLIENT_<OP>_CONTEXT_KEY`` in
|
||||
``opentelemetry.context``, LiteLLM:
|
||||
- Suppresses its duplicate client span, ``gen_ai.client.*`` metrics, and
|
||||
operation-exception log events.
|
||||
- Publishes its mapped span attributes and ``metric_attributes`` into the
|
||||
active context data container.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, cast
|
||||
|
||||
from opentelemetry.context import Context, get_value
|
||||
from opentelemetry.util.genai._invocation import _ContextData
|
||||
from opentelemetry.util.genai.invocation import (
|
||||
CLIENT_EMBEDDING_CONTEXT_KEY,
|
||||
CLIENT_FETCH_RESPONSE_CONTEXT_KEY,
|
||||
CLIENT_INFERENCE_CONTEXT_KEY,
|
||||
CLIENT_RETRIEVAL_CONTEXT_KEY,
|
||||
EmbeddingData,
|
||||
FetchResponseData,
|
||||
InferenceData,
|
||||
RetrievalData,
|
||||
)
|
||||
from opentelemetry.util.types import AttributeValue
|
||||
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData
|
||||
from litellm.integrations.otel.model.semconv import Error, GenAIOperation
|
||||
|
||||
__all__ = [
|
||||
"CLIENT_EMBEDDING_CONTEXT_KEY",
|
||||
"CLIENT_FETCH_RESPONSE_CONTEXT_KEY",
|
||||
"CLIENT_INFERENCE_CONTEXT_KEY",
|
||||
"CLIENT_RETRIEVAL_CONTEXT_KEY",
|
||||
"EmbeddingData",
|
||||
"FetchResponseData",
|
||||
"InferenceData",
|
||||
"RetrievalData",
|
||||
"context_spec_for_operation",
|
||||
"get_active_genai_context_data",
|
||||
"publish_to_genai_context",
|
||||
]
|
||||
|
||||
_INFERENCE_OPERATIONS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
GenAIOperation.CHAT.value,
|
||||
GenAIOperation.TEXT_COMPLETION.value,
|
||||
GenAIOperation.GENERATE_CONTENT.value,
|
||||
}
|
||||
)
|
||||
|
||||
_FETCH_RESPONSE_CALL_TYPES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"get_responses",
|
||||
"aget_responses",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def context_spec_for_operation(
|
||||
operation: GenAIOperation | str | None,
|
||||
call_type: str | None = None,
|
||||
) -> tuple[str, type[_ContextData]] | None:
|
||||
"""Return ``(context_key, dataclass_type)`` for a GenAI client operation, or ``None``."""
|
||||
op_str: Final = operation.value if isinstance(operation, GenAIOperation) else (operation or "")
|
||||
if call_type and call_type.lower() in _FETCH_RESPONSE_CALL_TYPES:
|
||||
return CLIENT_FETCH_RESPONSE_CONTEXT_KEY, FetchResponseData
|
||||
if op_str == "fetch_response":
|
||||
return CLIENT_FETCH_RESPONSE_CONTEXT_KEY, FetchResponseData
|
||||
if op_str in _INFERENCE_OPERATIONS:
|
||||
return CLIENT_INFERENCE_CONTEXT_KEY, InferenceData
|
||||
if op_str == GenAIOperation.EMBEDDINGS.value:
|
||||
return CLIENT_EMBEDDING_CONTEXT_KEY, EmbeddingData
|
||||
if op_str == GenAIOperation.RETRIEVAL.value:
|
||||
return CLIENT_RETRIEVAL_CONTEXT_KEY, RetrievalData
|
||||
return None
|
||||
|
||||
|
||||
def get_active_genai_context_data(
|
||||
operation: GenAIOperation | str | None,
|
||||
*,
|
||||
call_type: str | None = None,
|
||||
context: Context | None = None,
|
||||
) -> _ContextData | None:
|
||||
"""Return the active outer ``<Op>Data`` from ``opentelemetry.context`` if present."""
|
||||
spec: Final = context_spec_for_operation(operation, call_type=call_type)
|
||||
if spec is None:
|
||||
return None
|
||||
context_key, data_cls = spec
|
||||
for candidate_ctx in (context, None):
|
||||
value = get_value(context_key, context=candidate_ctx)
|
||||
if isinstance(value, data_cls):
|
||||
return value
|
||||
if candidate_ctx is None:
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def publish_to_genai_context(
|
||||
ctx_data: _ContextData,
|
||||
data: LLMCallSpanData,
|
||||
span_attrs: Mapping[str, object],
|
||||
metric_attrs: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
"""Publish LiteLLM's mapped span and metric attributes into an outer ``ctx_data``."""
|
||||
for key, val in span_attrs.items():
|
||||
if val is not None and isinstance(val, (str, bool, int, float, list, tuple)):
|
||||
ctx_data.attributes[key] = cast(AttributeValue, val)
|
||||
|
||||
if data.error is not None:
|
||||
ctx_data.attributes[Error.TYPE] = data.error.error_type
|
||||
if data.error.message:
|
||||
ctx_data.attributes[Error.MESSAGE] = data.error.message
|
||||
|
||||
if metric_attrs:
|
||||
for key, val in metric_attrs.items():
|
||||
if val is not None and isinstance(val, (str, bool, int, float)):
|
||||
ctx_data.metric_attributes[key] = val
|
||||
|
|
@ -29,6 +29,10 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
|
||||
from litellm.integrations.otel.genai_context import (
|
||||
get_active_genai_context_data,
|
||||
publish_to_genai_context,
|
||||
)
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
from litellm.integrations.otel.model.baggage import promoted_baggage
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config, excluded_db_systems_from
|
||||
|
|
@ -51,7 +55,7 @@ from litellm.integrations.otel.model.payloads import (
|
|||
)
|
||||
from litellm.integrations.otel.model.semconv import Error
|
||||
from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
|
||||
from litellm.integrations.otel.model.utils import to_ns
|
||||
from litellm.integrations.otel.model.utils import as_str, to_ns
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
active_phase,
|
||||
active_phase_span,
|
||||
|
|
@ -169,17 +173,19 @@ class _LLMCallSpan:
|
|||
under it, and must be released exactly once when the carrier is removed.
|
||||
"""
|
||||
|
||||
__slots__ = ("provider", "span", "start_time_ns")
|
||||
__slots__ = ("outer_ctx_data", "provider", "span", "start_time_ns")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
span: "Span | None",
|
||||
start_time_ns: int | None,
|
||||
provider: "TracerProvider | None" = None,
|
||||
outer_ctx_data: "object | None" = None,
|
||||
) -> None:
|
||||
self.span = span
|
||||
self.start_time_ns = start_time_ns
|
||||
self.provider = provider
|
||||
self.outer_ctx_data = outer_ctx_data
|
||||
|
||||
|
||||
def _llm_call_parent_context(call: LLMCallEvent) -> Context:
|
||||
|
|
@ -332,6 +338,17 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# Do not route on the deferred path: creating or LRU-touching a tenant
|
||||
# provider here would evict idle ones even though close re-routes.
|
||||
parent_context: Final = _llm_call_parent_context(call)
|
||||
outer_ctx_data: Final = get_active_genai_context_data(
|
||||
call.operation,
|
||||
call_type=as_str(kwargs.get("call_type")),
|
||||
context=parent_context,
|
||||
)
|
||||
if outer_ctx_data is not None:
|
||||
self._store_open_call(
|
||||
call_id,
|
||||
_LLMCallSpan(span=None, start_time_ns=start_time_ns, outer_ctx_data=outer_ctx_data),
|
||||
)
|
||||
return
|
||||
if not is_recordable_span(get_current_span(parent_context)):
|
||||
self._store_open_call(call_id, _LLMCallSpan(span=None, start_time_ns=start_time_ns))
|
||||
return
|
||||
|
|
@ -368,8 +385,9 @@ class OpenTelemetryV2(CustomLogger):
|
|||
return
|
||||
if self._emit_mcp_list_tools(kwargs, start_time, end_time):
|
||||
return
|
||||
self._close_llm_call(kwargs, start_time, end_time)
|
||||
self._record_metrics(kwargs, response_obj, start_time, end_time)
|
||||
_, suppressed = self._close_llm_call_with_suppression(kwargs, start_time, end_time)
|
||||
if not suppressed:
|
||||
self._record_metrics(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
def _record_metrics(self, kwargs, response_obj, start_time, end_time) -> None:
|
||||
"""Record the GenAI metrics for a successful LLM call."""
|
||||
|
|
@ -416,8 +434,9 @@ class OpenTelemetryV2(CustomLogger):
|
|||
return
|
||||
if self._emit_mcp_list_tools(kwargs, start_time, end_time):
|
||||
return
|
||||
self._close_llm_call(kwargs, start_time, end_time)
|
||||
self._record_failure_metrics(kwargs, start_time, end_time)
|
||||
_, suppressed = self._close_llm_call_with_suppression(kwargs, start_time, end_time)
|
||||
if not suppressed:
|
||||
self._record_failure_metrics(kwargs, start_time, end_time)
|
||||
|
||||
def _seed_identity_baggage(self, identity: RequestIdentity, model: str | None, context: Context) -> Context:
|
||||
"""Seed authenticated request-identity Baggage onto ``context`` so the Baggage
|
||||
|
|
@ -526,17 +545,25 @@ class OpenTelemetryV2(CustomLogger):
|
|||
start_time: datetime | float | None,
|
||||
end_time: datetime | float | None,
|
||||
) -> Span | None:
|
||||
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred).
|
||||
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred)."""
|
||||
span, _ = self._close_llm_call_with_suppression(kwargs, start_time, end_time)
|
||||
return span
|
||||
|
||||
No carrier for this call id means ``pre_call`` never ran — the request was
|
||||
rejected at the gate or blocked by a pre-call guardrail before any upstream
|
||||
call — so there is nothing to record and no phantom span.
|
||||
"""
|
||||
def _close_llm_call_with_suppression(
|
||||
self,
|
||||
kwargs: Mapping[str, object],
|
||||
start_time: datetime | float | None,
|
||||
end_time: datetime | float | None,
|
||||
) -> tuple[Span | None, bool]:
|
||||
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred),
|
||||
returning ``(span, suppressed)`` where ``suppressed`` is True when an outer
|
||||
GenAI instrumentation owned the operation."""
|
||||
call: Final = LLMCallEvent.from_dict(kwargs)
|
||||
call_id: Final = call.call_id
|
||||
# ``pop`` is the dedup: this method runs from both the success and failure
|
||||
# paths, and whichever fires first removes the carrier and closes the span.
|
||||
carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
|
||||
outer_ctx_data: Final = self._resolve_outer_genai_context(carrier, call, kwargs)
|
||||
# A missing carrier does not always mean nothing happened: a team/key-scoped
|
||||
# logger is a success/failure callback only, so ``pre_call`` never reaches it
|
||||
# and no carrier exists. The payload plus the request-level provider-handoff
|
||||
|
|
@ -544,15 +571,37 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
|
||||
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
|
||||
# upstream stamp, since the payload proves the call completed.
|
||||
if carrier is None and not call.reached_a_provider:
|
||||
return None
|
||||
if carrier is None and not call.reached_a_provider and outer_ctx_data is None:
|
||||
return None, False
|
||||
try:
|
||||
return self._finish_carrier(carrier, call, start_time, end_time)
|
||||
span = self._finish_carrier(
|
||||
carrier,
|
||||
call,
|
||||
start_time,
|
||||
end_time,
|
||||
kwargs=kwargs,
|
||||
outer_ctx_data=outer_ctx_data,
|
||||
)
|
||||
return span, outer_ctx_data is not None
|
||||
finally:
|
||||
# After the span has ended, so a release-triggered provider shutdown
|
||||
# force-flushes it out rather than racing its enqueue.
|
||||
self._release_carrier(carrier)
|
||||
|
||||
def _resolve_outer_genai_context(
|
||||
self,
|
||||
carrier: "_LLMCallSpan | None",
|
||||
call: LLMCallEvent,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> object | None:
|
||||
if carrier is not None and carrier.outer_ctx_data is not None:
|
||||
return carrier.outer_ctx_data
|
||||
return get_active_genai_context_data(
|
||||
call.operation,
|
||||
call_type=as_str(kwargs.get("call_type")),
|
||||
context=_llm_call_parent_context(call),
|
||||
)
|
||||
|
||||
def _store_open_call(self, call_id: str, carrier: _LLMCallSpan) -> None:
|
||||
"""Remember an in-flight LLM call, evicting the oldest if over budget.
|
||||
|
||||
|
|
@ -579,6 +628,9 @@ class OpenTelemetryV2(CustomLogger):
|
|||
call: LLMCallEvent,
|
||||
start_time: datetime | float | None,
|
||||
end_time: datetime | float | None,
|
||||
*,
|
||||
kwargs: Mapping[str, object] | None = None,
|
||||
outer_ctx_data: object | None = None,
|
||||
) -> Span | None:
|
||||
payload: Final = call.payload
|
||||
call_id: Final = call.call_id
|
||||
|
|
@ -603,6 +655,22 @@ class OpenTelemetryV2(CustomLogger):
|
|||
metadata_keys=tuple(self.config.baggage_metadata_keys),
|
||||
operation=call.operation,
|
||||
)
|
||||
if outer_ctx_data is not None:
|
||||
if self._emitter._seen(call_id, SpanRole.LLM_CALL):
|
||||
return None
|
||||
mapped: Final = self._emitter.map_attributes(data)
|
||||
metric_attrs: Final = (
|
||||
self._metrics_recorder.bounded_attributes_for_context(kwargs)
|
||||
if self._metrics_recorder is not None and kwargs is not None
|
||||
else None
|
||||
)
|
||||
publish_to_genai_context(
|
||||
outer_ctx_data, # type: ignore[arg-type]
|
||||
data,
|
||||
mapped,
|
||||
metric_attrs,
|
||||
)
|
||||
return None
|
||||
end_time_ns: Final = to_ns(end_time)
|
||||
if carrier is not None and carrier.span is not None:
|
||||
# Born at the boundary: stamp attributes from the typed payload, set
|
||||
|
|
|
|||
|
|
@ -366,6 +366,11 @@ class GenAIMetricRecorder:
|
|||
return {k: v for k, v in attrs.items() if k not in self._exclude}
|
||||
return attrs
|
||||
|
||||
def bounded_attributes_for_context(self, kwargs: Mapping[str, object]) -> dict[str, str]:
|
||||
"""Return bounded, filtered metric attributes suitable for publishing to an outer GenAI context data object."""
|
||||
filtered = self._filter_attributes(self._bounded_attributes(kwargs))
|
||||
return {k: v for k, v in filtered.items() if isinstance(v, str) and v}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Per-metric recording
|
||||
# ------------------------------------------------------------------ #
|
||||
|
|
|
|||
|
|
@ -185,6 +185,7 @@ proxy-runtime = [
|
|||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"opentelemetry-util-genai>=1.4b0",
|
||||
"ddtrace>=4.8.2,<5.0",
|
||||
"sentry-sdk>=2.21.0,<3.0",
|
||||
"mangum>=0.17.0,<1.0",
|
||||
|
|
@ -231,6 +232,7 @@ dev = [
|
|||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"opentelemetry-util-genai>=1.4b0",
|
||||
"langfuse>=4.7,<5.0",
|
||||
"fastapi-offline==1.7.6",
|
||||
"fakeredis==2.34.1",
|
||||
|
|
@ -263,6 +265,7 @@ proxy-dev = [
|
|||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"opentelemetry-util-genai>=1.4b0",
|
||||
"azure-identity==1.25.2",
|
||||
"a2a-sdk==1.1.0",
|
||||
"backports-zstd==1.8.0; python_version < '3.14'",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,407 @@
|
|||
"""Tests for OpenTelemetry GenAI client suppression and context attribute propagation.
|
||||
|
||||
Covers LiteLLM as the inner layer per open-telemetry/semantic-conventions-genai#682 and
|
||||
open-telemetry/opentelemetry-python-genai#859:
|
||||
When an outer framework instrumentation (e.g. LangChain, CrewAI) has already
|
||||
attached an ``InferenceData`` / ``EmbeddingData`` in ``opentelemetry.context``
|
||||
under ``CLIENT_<OP>_CONTEXT_KEY``, LiteLLM suppresses its duplicate client span,
|
||||
metrics, and events and publishes its resolved attributes into the active
|
||||
context data object.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from opentelemetry.context import attach, detach, set_value
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
|
||||
from litellm.integrations.otel.genai_context import (
|
||||
CLIENT_EMBEDDING_CONTEXT_KEY,
|
||||
CLIENT_INFERENCE_CONTEXT_KEY,
|
||||
EmbeddingData,
|
||||
InferenceData,
|
||||
)
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.semconv import Error, GenAI, LiteLLM, Server
|
||||
from litellm.types.utils import StandardLoggingModelInformation, StandardLoggingPayload
|
||||
|
||||
|
||||
def _make_payload(
|
||||
*,
|
||||
call_id: str = "call-123",
|
||||
call_type: str = "completion",
|
||||
model: str = "gpt-4o",
|
||||
custom_llm_provider: str = "openai",
|
||||
api_base: str = "https://api.openai.com:443/v1",
|
||||
prompt_tokens: int = 12,
|
||||
completion_tokens: int = 8,
|
||||
response_cost: float = 0.0015,
|
||||
status: str = "success",
|
||||
error_str: str | None = None,
|
||||
error_information: dict | None = None,
|
||||
) -> StandardLoggingPayload:
|
||||
now = datetime.now(timezone.utc)
|
||||
return StandardLoggingPayload(
|
||||
id="chatcmpl-xyz789",
|
||||
trace_id="trace-1",
|
||||
call_type=call_type,
|
||||
stream=False,
|
||||
response_cost=response_cost,
|
||||
status=status, # type: ignore[typeddict-item]
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
startTime=now.timestamp(),
|
||||
endTime=(now + timedelta(milliseconds=250)).timestamp(),
|
||||
completionStartTime=now.timestamp(),
|
||||
response_time=0.25,
|
||||
model_map_information=StandardLoggingModelInformation(
|
||||
model_map_key=model,
|
||||
model_map_value=model,
|
||||
),
|
||||
model=model,
|
||||
model_id="deployment-1",
|
||||
model_group=model,
|
||||
api_base=api_base,
|
||||
metadata={
|
||||
"user_api_key_hash": "hash-abc",
|
||||
"user_api_key_team_id": "team-1",
|
||||
"user_api_key_user_id": None,
|
||||
"user_api_key_org_id": None,
|
||||
"user_api_key_team_alias": "core-team",
|
||||
"user_api_key_alias": "key-alias",
|
||||
"user_api_key_user_email": None,
|
||||
"user_api_key_end_user_id": None,
|
||||
"spend_logs_metadata": None,
|
||||
"requester_ip_address": None,
|
||||
"requester_metadata": None,
|
||||
"requester_custom_headers": None,
|
||||
"prompt_management_metadata": None,
|
||||
"applied_guardrails": None,
|
||||
"mcp_tool_call_metadata": None,
|
||||
"vector_store_request_metadata": None,
|
||||
"usage_object": None,
|
||||
"model_info": None,
|
||||
"guardrail_information": None,
|
||||
},
|
||||
cache_hit=False,
|
||||
cache_key=None,
|
||||
saved_cache_cost=0.0,
|
||||
request_tags=[],
|
||||
end_user=None,
|
||||
requester_ip_address=None,
|
||||
messages=[{"role": "user", "content": "Hello world"}],
|
||||
response={
|
||||
"id": "chatcmpl-xyz789",
|
||||
"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hi there!"}}],
|
||||
},
|
||||
error_str=error_str,
|
||||
error_information=error_information, # type: ignore[typeddict-item]
|
||||
model_parameters={
|
||||
"temperature": 0.5,
|
||||
"max_tokens": 100,
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather", "description": "Get weather"}}],
|
||||
},
|
||||
hidden_params={"model_id": "deployment-1", "litellm_model_name": model},
|
||||
litellm_call_id=call_id,
|
||||
)
|
||||
|
||||
|
||||
def _make_v2_harness(*, capture_span_content: bool = False):
|
||||
span_exporter = InMemorySpanExporter()
|
||||
tracer_provider = TracerProvider()
|
||||
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
|
||||
|
||||
metric_reader = InMemoryMetricReader()
|
||||
meter_provider = MeterProvider(metric_readers=[metric_reader])
|
||||
|
||||
log_exporter = InMemoryLogExporter()
|
||||
logger_provider = LoggerProvider()
|
||||
logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
|
||||
|
||||
cfg = OpenTelemetryV2Config(
|
||||
exporter="in_memory",
|
||||
enable_metrics=True,
|
||||
enable_events=True,
|
||||
capture_message_content="span_and_event" if capture_span_content else "no_content",
|
||||
)
|
||||
logger = OpenTelemetryV2(
|
||||
config=cfg,
|
||||
tracer_provider=tracer_provider,
|
||||
meter_provider=meter_provider,
|
||||
logger_provider=logger_provider,
|
||||
)
|
||||
return logger, span_exporter, metric_reader, log_exporter, tracer_provider, meter_provider, logger_provider
|
||||
|
||||
|
||||
def _collected_metric_names(metric_reader: InMemoryMetricReader) -> list[str]:
|
||||
data = metric_reader.get_metrics_data()
|
||||
if data is None:
|
||||
return []
|
||||
names: list[str] = []
|
||||
for rm in data.resource_metrics:
|
||||
for sm in rm.scope_metrics:
|
||||
for metric in sm.metrics:
|
||||
names.append(metric.name)
|
||||
return names
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
# 1. LiteLLM V2 as Inner Layer (Suppressed by Outer GenAI Context)
|
||||
# =========================================================================== #
|
||||
|
||||
|
||||
def test_v2_suppressed_when_outer_inference_context_active():
|
||||
"""When CLIENT_INFERENCE_CONTEXT_KEY holds an InferenceData, OpenTelemetryV2
|
||||
suppresses its own client span and metrics and enriches the outer InferenceData."""
|
||||
logger, span_exporter, metric_reader, log_exporter, *_ = _make_v2_harness(capture_span_content=True)
|
||||
|
||||
outer_data = InferenceData(request_model="langchain-alias")
|
||||
token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data))
|
||||
try:
|
||||
call_id = "call-suppressed-v2"
|
||||
payload = _make_payload(call_id=call_id, model="gpt-4o", custom_llm_provider="openai")
|
||||
kwargs = {
|
||||
"litellm_call_id": call_id,
|
||||
"call_type": "completion",
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "Hello world"}],
|
||||
"optional_params": {
|
||||
"temperature": 0.5,
|
||||
"max_tokens": 100,
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather", "description": "Get weather"}}],
|
||||
},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "metadata": {}},
|
||||
"standard_logging_object": payload,
|
||||
"response_cost": 0.0015,
|
||||
}
|
||||
response_obj = {
|
||||
"id": "chatcmpl-xyz789",
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hi there!"}}],
|
||||
"usage": {"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20},
|
||||
}
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
logger.log_pre_api_call("gpt-4o", kwargs["messages"], kwargs)
|
||||
asyncio.run(
|
||||
logger.async_log_success_event(
|
||||
kwargs,
|
||||
response_obj,
|
||||
now,
|
||||
now + timedelta(milliseconds=250),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
detach(token)
|
||||
|
||||
# 1. No span, metrics, or events emitted by LiteLLM
|
||||
assert span_exporter.get_finished_spans() == ()
|
||||
assert _collected_metric_names(metric_reader) == []
|
||||
assert log_exporter.get_finished_logs() == ()
|
||||
|
||||
# 2. Outer InferenceData is enriched via attributes and metric_attributes dicts
|
||||
assert outer_data.attributes.get(GenAI.PROVIDER_NAME) == "openai"
|
||||
assert outer_data.attributes.get(GenAI.OPERATION_NAME) == "chat"
|
||||
assert outer_data.attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
|
||||
assert outer_data.attributes.get(GenAI.REQUEST_TEMPERATURE) == 0.5
|
||||
assert outer_data.attributes.get(GenAI.REQUEST_MAX_TOKENS) == 100
|
||||
assert outer_data.attributes.get(GenAI.RESPONSE_ID) == "chatcmpl-xyz789"
|
||||
assert list(outer_data.attributes.get(GenAI.RESPONSE_FINISH_REASONS) or []) == ["stop"]
|
||||
assert outer_data.attributes.get(GenAI.USAGE_INPUT_TOKENS) == 12
|
||||
assert outer_data.attributes.get(GenAI.USAGE_OUTPUT_TOKENS) == 8
|
||||
assert outer_data.attributes.get(Server.ADDRESS) == "api.openai.com"
|
||||
assert outer_data.attributes.get(Server.PORT) == 443
|
||||
|
||||
# Custom LiteLLM attributes published to outer_data.attributes
|
||||
assert outer_data.attributes.get(LiteLLM.CALL_ID) == call_id
|
||||
assert outer_data.attributes.get(LiteLLM.CACHE_HIT) is False
|
||||
|
||||
# Metric attributes published to outer_data.metric_attributes
|
||||
assert outer_data.metric_attributes.get(GenAI.PROVIDER_NAME) == "openai"
|
||||
assert outer_data.metric_attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
|
||||
|
||||
# Content capture attributes published into outer_data.attributes
|
||||
assert outer_data.attributes.get(GenAI.INPUT_MESSAGES) is not None
|
||||
assert outer_data.attributes.get(GenAI.OUTPUT_MESSAGES) is not None
|
||||
|
||||
|
||||
def test_v2_suppressed_failure_enriches_outer_inference_context():
|
||||
"""On a failed LLM call when suppressed, OpenTelemetryV2 suppresses its error span,
|
||||
failure duration metric, and exception event, and publishes error details to outer_data."""
|
||||
logger, span_exporter, metric_reader, log_exporter, *_ = _make_v2_harness()
|
||||
|
||||
outer_data = InferenceData()
|
||||
token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data))
|
||||
try:
|
||||
call_id = "call-fail-suppressed-v2"
|
||||
payload = _make_payload(
|
||||
call_id=call_id,
|
||||
model="gpt-4o",
|
||||
custom_llm_provider="openai",
|
||||
status="failure",
|
||||
error_str="Rate limit exceeded",
|
||||
error_information={
|
||||
"error_code": "429",
|
||||
"error_class": "RateLimitError",
|
||||
"error_message": "Rate limit exceeded",
|
||||
"llm_provider": "openai",
|
||||
"traceback": "Traceback ...",
|
||||
},
|
||||
)
|
||||
kwargs = {
|
||||
"litellm_call_id": call_id,
|
||||
"call_type": "completion",
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"optional_params": {},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "metadata": {}},
|
||||
"standard_logging_object": payload,
|
||||
"exception": RuntimeError("Rate limit exceeded"),
|
||||
}
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
logger.log_pre_api_call("gpt-4o", kwargs["messages"], kwargs)
|
||||
asyncio.run(
|
||||
logger.async_log_failure_event(
|
||||
kwargs,
|
||||
None,
|
||||
now,
|
||||
now + timedelta(milliseconds=100),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
detach(token)
|
||||
|
||||
assert span_exporter.get_finished_spans() == ()
|
||||
assert _collected_metric_names(metric_reader) == []
|
||||
assert log_exporter.get_finished_logs() == ()
|
||||
|
||||
assert outer_data.attributes.get(Error.TYPE) == "RateLimitError"
|
||||
assert outer_data.attributes.get(Error.MESSAGE) == "Rate limit exceeded"
|
||||
|
||||
|
||||
def test_v2_suppressed_with_real_opentelemetry_util_genai_handler():
|
||||
"""End-to-end test with opentelemetry.util.genai.handler.TelemetryHandler as the outer layer."""
|
||||
util_genai = pytest.importorskip("opentelemetry.util.genai.handler")
|
||||
|
||||
(
|
||||
logger,
|
||||
span_exporter,
|
||||
_metric_reader,
|
||||
_log_exporter,
|
||||
tracer_provider,
|
||||
meter_provider,
|
||||
logger_provider,
|
||||
) = _make_v2_harness(capture_span_content=True)
|
||||
|
||||
handler = util_genai.TelemetryHandler(
|
||||
tracer_provider=tracer_provider,
|
||||
meter_provider=meter_provider,
|
||||
logger_provider=logger_provider,
|
||||
)
|
||||
|
||||
call_id = "call-e2e-outer-handler"
|
||||
payload = _make_payload(
|
||||
call_id=call_id,
|
||||
model="claude-3-5-sonnet",
|
||||
custom_llm_provider="anthropic",
|
||||
prompt_tokens=15,
|
||||
completion_tokens=9,
|
||||
)
|
||||
kwargs = {
|
||||
"litellm_call_id": call_id,
|
||||
"call_type": "completion",
|
||||
"model": "claude-3-5-sonnet",
|
||||
"messages": [{"role": "user", "content": "Hello Claude"}],
|
||||
"optional_params": {"temperature": 0.2},
|
||||
"litellm_params": {"custom_llm_provider": "anthropic", "metadata": {}},
|
||||
"standard_logging_object": payload,
|
||||
"response_cost": 0.003,
|
||||
}
|
||||
response_obj = {
|
||||
"id": "msg-01",
|
||||
"model": "claude-3-5-sonnet-20241022",
|
||||
"choices": [{"finish_reason": "end_turn", "message": {"role": "assistant", "content": "Hello!"}}],
|
||||
"usage": {"prompt_tokens": 15, "completion_tokens": 9, "total_tokens": 24},
|
||||
}
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
with handler.inference(None, request_model="outer-model"):
|
||||
logger.log_pre_api_call("claude-3-5-sonnet", kwargs["messages"], kwargs)
|
||||
asyncio.run(
|
||||
logger.async_log_success_event(
|
||||
kwargs,
|
||||
response_obj,
|
||||
now,
|
||||
now + timedelta(milliseconds=200),
|
||||
)
|
||||
)
|
||||
|
||||
spans = span_exporter.get_finished_spans()
|
||||
# Exactly ONE client span is emitted (by the outer TelemetryHandler, enriched by LiteLLM)
|
||||
assert len(spans) == 1
|
||||
outer_span = spans[0]
|
||||
attrs = dict(outer_span.attributes or {})
|
||||
assert attrs.get(GenAI.PROVIDER_NAME) == "anthropic"
|
||||
assert attrs.get(GenAI.USAGE_INPUT_TOKENS) == 15
|
||||
assert attrs.get(GenAI.USAGE_OUTPUT_TOKENS) == 9
|
||||
assert attrs.get(LiteLLM.CALL_ID) == call_id
|
||||
|
||||
|
||||
def test_v2_embedding_context_suppression():
|
||||
"""Embedding calls use CLIENT_EMBEDDING_CONTEXT_KEY and EmbeddingData."""
|
||||
logger, span_exporter, metric_reader, *_ = _make_v2_harness()
|
||||
|
||||
outer_emb = EmbeddingData(request_model="text-embedding-3-small")
|
||||
token = attach(set_value(CLIENT_EMBEDDING_CONTEXT_KEY, outer_emb))
|
||||
try:
|
||||
call_id = "call-emb-suppressed"
|
||||
payload = _make_payload(
|
||||
call_id=call_id,
|
||||
call_type="embedding",
|
||||
model="text-embedding-3-small",
|
||||
custom_llm_provider="openai",
|
||||
prompt_tokens=10,
|
||||
completion_tokens=0,
|
||||
)
|
||||
kwargs = {
|
||||
"litellm_call_id": call_id,
|
||||
"call_type": "embedding",
|
||||
"model": "text-embedding-3-small",
|
||||
"messages": ["embed me"],
|
||||
"optional_params": {"encoding_format": "float"},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "metadata": {}},
|
||||
"standard_logging_object": payload,
|
||||
}
|
||||
now = datetime.now(timezone.utc)
|
||||
logger.log_pre_api_call("text-embedding-3-small", kwargs["messages"], kwargs)
|
||||
asyncio.run(
|
||||
logger.async_log_success_event(
|
||||
kwargs,
|
||||
None,
|
||||
now,
|
||||
now + timedelta(milliseconds=100),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
detach(token)
|
||||
|
||||
assert span_exporter.get_finished_spans() == ()
|
||||
assert _collected_metric_names(metric_reader) == []
|
||||
assert outer_emb.attributes.get(GenAI.PROVIDER_NAME) == "openai"
|
||||
assert outer_emb.attributes.get(GenAI.REQUEST_MODEL) == "text-embedding-3-small"
|
||||
assert outer_emb.attributes.get(GenAI.USAGE_INPUT_TOKENS) == 10
|
||||
|
||||
|
||||
Loading…
Add table
Reference in a new issue