feat(otel): support GenAI client suppression and context attribute propagation

This commit is contained in:
Dylan Russell 2026-11-04 17:09:49 +00:00
parent 03c26c6e07
commit c0d506b557
6 changed files with 645 additions and 26 deletions

View file

@ -228,6 +228,25 @@ class SpanEmitter:
# -- the engine ---------------------------------------------------------- #
def _map_data(self, data: SpanData) -> tuple[dict[str, AttrValue], Mapping[str, AttrValue]]:
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
for_langfuse: Final = (
dict(
sorted(
self._langfuse.map(data).items(),
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
)
)
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
else _NO_ATTRIBUTES
)
return {**configured, **for_langfuse}, for_langfuse
def map_attributes(self, data: SpanData) -> dict[str, AttrValue]:
"""Run the configured mapper chain on ``data`` without creating a span."""
mapped, _ = self._map_data(data)
return mapped
def emit(
self,
role: SpanRole,
@ -299,18 +318,8 @@ class SpanEmitter:
)
else None
)
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
for_langfuse: Final = (
dict(
sorted(
self._langfuse.map(data).items(),
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
)
)
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
else _NO_ATTRIBUTES
)
mapped: Final = MappingProxyType({**configured, **for_langfuse})
raw_mapped, for_langfuse = self._map_data(data)
mapped: Final = MappingProxyType(raw_mapped)
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
reserved: Final = len(stamped_later.keys() - mapped.keys())
overwrites: Final = carried_keys(span) & mapped.keys()

View file

@ -0,0 +1,127 @@
"""OpenTelemetry GenAI client context suppression & attribute propagation.
Integrates ``OpenTelemetryV2`` with ``opentelemetry-util-genai``'s context-sharing
contract (see ``open-telemetry/semantic-conventions-genai#682`` and
``open-telemetry/opentelemetry-python-genai#859``):
When LiteLLM is called inside an outer GenAI instrumentation that attached
``InferenceData``, ``EmbeddingData``, ``RetrievalData``, or
``FetchResponseData`` under ``CLIENT_<OP>_CONTEXT_KEY`` in
``opentelemetry.context``, LiteLLM:
- Suppresses its duplicate client span, ``gen_ai.client.*`` metrics, and
operation-exception log events.
- Publishes its mapped span attributes and ``metric_attributes`` into the
active context data container.
"""
from __future__ import annotations
from collections.abc import Mapping
from typing import Final, cast
from opentelemetry.context import Context, get_value
from opentelemetry.util.genai._invocation import _ContextData
from opentelemetry.util.genai.invocation import (
CLIENT_EMBEDDING_CONTEXT_KEY,
CLIENT_FETCH_RESPONSE_CONTEXT_KEY,
CLIENT_INFERENCE_CONTEXT_KEY,
CLIENT_RETRIEVAL_CONTEXT_KEY,
EmbeddingData,
FetchResponseData,
InferenceData,
RetrievalData,
)
from opentelemetry.util.types import AttributeValue
from litellm.integrations.otel.model.payloads import LLMCallSpanData
from litellm.integrations.otel.model.semconv import Error, GenAIOperation
__all__ = [
"CLIENT_EMBEDDING_CONTEXT_KEY",
"CLIENT_FETCH_RESPONSE_CONTEXT_KEY",
"CLIENT_INFERENCE_CONTEXT_KEY",
"CLIENT_RETRIEVAL_CONTEXT_KEY",
"EmbeddingData",
"FetchResponseData",
"InferenceData",
"RetrievalData",
"context_spec_for_operation",
"get_active_genai_context_data",
"publish_to_genai_context",
]
_INFERENCE_OPERATIONS: Final[frozenset[str]] = frozenset(
{
GenAIOperation.CHAT.value,
GenAIOperation.TEXT_COMPLETION.value,
GenAIOperation.GENERATE_CONTENT.value,
}
)
_FETCH_RESPONSE_CALL_TYPES: Final[frozenset[str]] = frozenset(
{
"get_responses",
"aget_responses",
}
)
def context_spec_for_operation(
operation: GenAIOperation | str | None,
call_type: str | None = None,
) -> tuple[str, type[_ContextData]] | None:
"""Return ``(context_key, dataclass_type)`` for a GenAI client operation, or ``None``."""
op_str: Final = operation.value if isinstance(operation, GenAIOperation) else (operation or "")
if call_type and call_type.lower() in _FETCH_RESPONSE_CALL_TYPES:
return CLIENT_FETCH_RESPONSE_CONTEXT_KEY, FetchResponseData
if op_str == "fetch_response":
return CLIENT_FETCH_RESPONSE_CONTEXT_KEY, FetchResponseData
if op_str in _INFERENCE_OPERATIONS:
return CLIENT_INFERENCE_CONTEXT_KEY, InferenceData
if op_str == GenAIOperation.EMBEDDINGS.value:
return CLIENT_EMBEDDING_CONTEXT_KEY, EmbeddingData
if op_str == GenAIOperation.RETRIEVAL.value:
return CLIENT_RETRIEVAL_CONTEXT_KEY, RetrievalData
return None
def get_active_genai_context_data(
operation: GenAIOperation | str | None,
*,
call_type: str | None = None,
context: Context | None = None,
) -> _ContextData | None:
"""Return the active outer ``<Op>Data`` from ``opentelemetry.context`` if present."""
spec: Final = context_spec_for_operation(operation, call_type=call_type)
if spec is None:
return None
context_key, data_cls = spec
for candidate_ctx in (context, None):
value = get_value(context_key, context=candidate_ctx)
if isinstance(value, data_cls):
return value
if candidate_ctx is None:
break
return None
def publish_to_genai_context(
ctx_data: _ContextData,
data: LLMCallSpanData,
span_attrs: Mapping[str, object],
metric_attrs: Mapping[str, object] | None = None,
) -> None:
"""Publish LiteLLM's mapped span and metric attributes into an outer ``ctx_data``."""
for key, val in span_attrs.items():
if val is not None and isinstance(val, (str, bool, int, float, list, tuple)):
ctx_data.attributes[key] = cast(AttributeValue, val)
if data.error is not None:
ctx_data.attributes[Error.TYPE] = data.error.error_type
if data.error.message:
ctx_data.attributes[Error.MESSAGE] = data.error.message
if metric_attrs:
for key, val in metric_attrs.items():
if val is not None and isinstance(val, (str, bool, int, float)):
ctx_data.metric_attributes[key] = val

View file

@ -29,6 +29,10 @@ import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
from litellm.integrations.otel.genai_context import (
get_active_genai_context_data,
publish_to_genai_context,
)
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.model.baggage import promoted_baggage
from litellm.integrations.otel.model.config import OpenTelemetryV2Config, excluded_db_systems_from
@ -51,7 +55,7 @@ from litellm.integrations.otel.model.payloads import (
)
from litellm.integrations.otel.model.semconv import Error
from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
from litellm.integrations.otel.model.utils import to_ns
from litellm.integrations.otel.model.utils import as_str, to_ns
from litellm.integrations.otel.plumbing.context import (
active_phase,
active_phase_span,
@ -169,17 +173,19 @@ class _LLMCallSpan:
under it, and must be released exactly once when the carrier is removed.
"""
__slots__ = ("provider", "span", "start_time_ns")
__slots__ = ("outer_ctx_data", "provider", "span", "start_time_ns")
def __init__(
self,
span: "Span | None",
start_time_ns: int | None,
provider: "TracerProvider | None" = None,
outer_ctx_data: "object | None" = None,
) -> None:
self.span = span
self.start_time_ns = start_time_ns
self.provider = provider
self.outer_ctx_data = outer_ctx_data
def _llm_call_parent_context(call: LLMCallEvent) -> Context:
@ -332,6 +338,17 @@ class OpenTelemetryV2(CustomLogger):
# Do not route on the deferred path: creating or LRU-touching a tenant
# provider here would evict idle ones even though close re-routes.
parent_context: Final = _llm_call_parent_context(call)
outer_ctx_data: Final = get_active_genai_context_data(
call.operation,
call_type=as_str(kwargs.get("call_type")),
context=parent_context,
)
if outer_ctx_data is not None:
self._store_open_call(
call_id,
_LLMCallSpan(span=None, start_time_ns=start_time_ns, outer_ctx_data=outer_ctx_data),
)
return
if not is_recordable_span(get_current_span(parent_context)):
self._store_open_call(call_id, _LLMCallSpan(span=None, start_time_ns=start_time_ns))
return
@ -368,8 +385,9 @@ class OpenTelemetryV2(CustomLogger):
return
if self._emit_mcp_list_tools(kwargs, start_time, end_time):
return
self._close_llm_call(kwargs, start_time, end_time)
self._record_metrics(kwargs, response_obj, start_time, end_time)
_, suppressed = self._close_llm_call_with_suppression(kwargs, start_time, end_time)
if not suppressed:
self._record_metrics(kwargs, response_obj, start_time, end_time)
def _record_metrics(self, kwargs, response_obj, start_time, end_time) -> None:
"""Record the GenAI metrics for a successful LLM call."""
@ -416,8 +434,9 @@ class OpenTelemetryV2(CustomLogger):
return
if self._emit_mcp_list_tools(kwargs, start_time, end_time):
return
self._close_llm_call(kwargs, start_time, end_time)
self._record_failure_metrics(kwargs, start_time, end_time)
_, suppressed = self._close_llm_call_with_suppression(kwargs, start_time, end_time)
if not suppressed:
self._record_failure_metrics(kwargs, start_time, end_time)
def _seed_identity_baggage(self, identity: RequestIdentity, model: str | None, context: Context) -> Context:
"""Seed authenticated request-identity Baggage onto ``context`` so the Baggage
@ -526,17 +545,25 @@ class OpenTelemetryV2(CustomLogger):
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> Span | None:
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred).
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred)."""
span, _ = self._close_llm_call_with_suppression(kwargs, start_time, end_time)
return span
No carrier for this call id means ``pre_call`` never ran — the request was
rejected at the gate or blocked by a pre-call guardrail before any upstream
call — so there is nothing to record and no phantom span.
"""
def _close_llm_call_with_suppression(
self,
kwargs: Mapping[str, object],
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> tuple[Span | None, bool]:
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred),
returning ``(span, suppressed)`` where ``suppressed`` is True when an outer
GenAI instrumentation owned the operation."""
call: Final = LLMCallEvent.from_dict(kwargs)
call_id: Final = call.call_id
# ``pop`` is the dedup: this method runs from both the success and failure
# paths, and whichever fires first removes the carrier and closes the span.
carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
outer_ctx_data: Final = self._resolve_outer_genai_context(carrier, call, kwargs)
# A missing carrier does not always mean nothing happened: a team/key-scoped
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
@ -544,15 +571,37 @@ class OpenTelemetryV2(CustomLogger):
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
# upstream stamp, since the payload proves the call completed.
if carrier is None and not call.reached_a_provider:
return None
if carrier is None and not call.reached_a_provider and outer_ctx_data is None:
return None, False
try:
return self._finish_carrier(carrier, call, start_time, end_time)
span = self._finish_carrier(
carrier,
call,
start_time,
end_time,
kwargs=kwargs,
outer_ctx_data=outer_ctx_data,
)
return span, outer_ctx_data is not None
finally:
# After the span has ended, so a release-triggered provider shutdown
# force-flushes it out rather than racing its enqueue.
self._release_carrier(carrier)
def _resolve_outer_genai_context(
self,
carrier: "_LLMCallSpan | None",
call: LLMCallEvent,
kwargs: Mapping[str, object],
) -> object | None:
if carrier is not None and carrier.outer_ctx_data is not None:
return carrier.outer_ctx_data
return get_active_genai_context_data(
call.operation,
call_type=as_str(kwargs.get("call_type")),
context=_llm_call_parent_context(call),
)
def _store_open_call(self, call_id: str, carrier: _LLMCallSpan) -> None:
"""Remember an in-flight LLM call, evicting the oldest if over budget.
@ -579,6 +628,9 @@ class OpenTelemetryV2(CustomLogger):
call: LLMCallEvent,
start_time: datetime | float | None,
end_time: datetime | float | None,
*,
kwargs: Mapping[str, object] | None = None,
outer_ctx_data: object | None = None,
) -> Span | None:
payload: Final = call.payload
call_id: Final = call.call_id
@ -603,6 +655,22 @@ class OpenTelemetryV2(CustomLogger):
metadata_keys=tuple(self.config.baggage_metadata_keys),
operation=call.operation,
)
if outer_ctx_data is not None:
if self._emitter._seen(call_id, SpanRole.LLM_CALL):
return None
mapped: Final = self._emitter.map_attributes(data)
metric_attrs: Final = (
self._metrics_recorder.bounded_attributes_for_context(kwargs)
if self._metrics_recorder is not None and kwargs is not None
else None
)
publish_to_genai_context(
outer_ctx_data, # type: ignore[arg-type]
data,
mapped,
metric_attrs,
)
return None
end_time_ns: Final = to_ns(end_time)
if carrier is not None and carrier.span is not None:
# Born at the boundary: stamp attributes from the typed payload, set

View file

@ -366,6 +366,11 @@ class GenAIMetricRecorder:
return {k: v for k, v in attrs.items() if k not in self._exclude}
return attrs
def bounded_attributes_for_context(self, kwargs: Mapping[str, object]) -> dict[str, str]:
"""Return bounded, filtered metric attributes suitable for publishing to an outer GenAI context data object."""
filtered = self._filter_attributes(self._bounded_attributes(kwargs))
return {k: v for k, v in filtered.items() if isinstance(v, str) and v}
# ------------------------------------------------------------------ #
# Per-metric recording
# ------------------------------------------------------------------ #

View file

@ -185,6 +185,7 @@ proxy-runtime = [
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"opentelemetry-util-genai>=1.4b0",
"ddtrace>=4.8.2,<5.0",
"sentry-sdk>=2.21.0,<3.0",
"mangum>=0.17.0,<1.0",
@ -231,6 +232,7 @@ dev = [
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"opentelemetry-util-genai>=1.4b0",
"langfuse>=4.7,<5.0",
"fastapi-offline==1.7.6",
"fakeredis==2.34.1",
@ -263,6 +265,7 @@ proxy-dev = [
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"opentelemetry-util-genai>=1.4b0",
"azure-identity==1.25.2",
"a2a-sdk==1.1.0",
"backports-zstd==1.8.0; python_version < '3.14'",

View file

@ -0,0 +1,407 @@
"""Tests for OpenTelemetry GenAI client suppression and context attribute propagation.
Covers LiteLLM as the inner layer per open-telemetry/semantic-conventions-genai#682 and
open-telemetry/opentelemetry-python-genai#859:
When an outer framework instrumentation (e.g. LangChain, CrewAI) has already
attached an ``InferenceData`` / ``EmbeddingData`` in ``opentelemetry.context``
under ``CLIENT_<OP>_CONTEXT_KEY``, LiteLLM suppresses its duplicate client span,
metrics, and events and publishes its resolved attributes into the active
context data object.
"""
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
import pytest
from opentelemetry.context import attach, detach, set_value
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from litellm.integrations.otel.genai_context import (
CLIENT_EMBEDDING_CONTEXT_KEY,
CLIENT_INFERENCE_CONTEXT_KEY,
EmbeddingData,
InferenceData,
)
from litellm.integrations.otel.logger import OpenTelemetryV2
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.model.semconv import Error, GenAI, LiteLLM, Server
from litellm.types.utils import StandardLoggingModelInformation, StandardLoggingPayload
def _make_payload(
*,
call_id: str = "call-123",
call_type: str = "completion",
model: str = "gpt-4o",
custom_llm_provider: str = "openai",
api_base: str = "https://api.openai.com:443/v1",
prompt_tokens: int = 12,
completion_tokens: int = 8,
response_cost: float = 0.0015,
status: str = "success",
error_str: str | None = None,
error_information: dict | None = None,
) -> StandardLoggingPayload:
now = datetime.now(timezone.utc)
return StandardLoggingPayload(
id="chatcmpl-xyz789",
trace_id="trace-1",
call_type=call_type,
stream=False,
response_cost=response_cost,
status=status, # type: ignore[typeddict-item]
custom_llm_provider=custom_llm_provider,
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
startTime=now.timestamp(),
endTime=(now + timedelta(milliseconds=250)).timestamp(),
completionStartTime=now.timestamp(),
response_time=0.25,
model_map_information=StandardLoggingModelInformation(
model_map_key=model,
model_map_value=model,
),
model=model,
model_id="deployment-1",
model_group=model,
api_base=api_base,
metadata={
"user_api_key_hash": "hash-abc",
"user_api_key_team_id": "team-1",
"user_api_key_user_id": None,
"user_api_key_org_id": None,
"user_api_key_team_alias": "core-team",
"user_api_key_alias": "key-alias",
"user_api_key_user_email": None,
"user_api_key_end_user_id": None,
"spend_logs_metadata": None,
"requester_ip_address": None,
"requester_metadata": None,
"requester_custom_headers": None,
"prompt_management_metadata": None,
"applied_guardrails": None,
"mcp_tool_call_metadata": None,
"vector_store_request_metadata": None,
"usage_object": None,
"model_info": None,
"guardrail_information": None,
},
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[{"role": "user", "content": "Hello world"}],
response={
"id": "chatcmpl-xyz789",
"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hi there!"}}],
},
error_str=error_str,
error_information=error_information, # type: ignore[typeddict-item]
model_parameters={
"temperature": 0.5,
"max_tokens": 100,
"tools": [{"type": "function", "function": {"name": "get_weather", "description": "Get weather"}}],
},
hidden_params={"model_id": "deployment-1", "litellm_model_name": model},
litellm_call_id=call_id,
)
def _make_v2_harness(*, capture_span_content: bool = False):
span_exporter = InMemorySpanExporter()
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
metric_reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[metric_reader])
log_exporter = InMemoryLogExporter()
logger_provider = LoggerProvider()
logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
cfg = OpenTelemetryV2Config(
exporter="in_memory",
enable_metrics=True,
enable_events=True,
capture_message_content="span_and_event" if capture_span_content else "no_content",
)
logger = OpenTelemetryV2(
config=cfg,
tracer_provider=tracer_provider,
meter_provider=meter_provider,
logger_provider=logger_provider,
)
return logger, span_exporter, metric_reader, log_exporter, tracer_provider, meter_provider, logger_provider
def _collected_metric_names(metric_reader: InMemoryMetricReader) -> list[str]:
data = metric_reader.get_metrics_data()
if data is None:
return []
names: list[str] = []
for rm in data.resource_metrics:
for sm in rm.scope_metrics:
for metric in sm.metrics:
names.append(metric.name)
return names
# =========================================================================== #
# 1. LiteLLM V2 as Inner Layer (Suppressed by Outer GenAI Context)
# =========================================================================== #
def test_v2_suppressed_when_outer_inference_context_active():
"""When CLIENT_INFERENCE_CONTEXT_KEY holds an InferenceData, OpenTelemetryV2
suppresses its own client span and metrics and enriches the outer InferenceData."""
logger, span_exporter, metric_reader, log_exporter, *_ = _make_v2_harness(capture_span_content=True)
outer_data = InferenceData(request_model="langchain-alias")
token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data))
try:
call_id = "call-suppressed-v2"
payload = _make_payload(call_id=call_id, model="gpt-4o", custom_llm_provider="openai")
kwargs = {
"litellm_call_id": call_id,
"call_type": "completion",
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello world"}],
"optional_params": {
"temperature": 0.5,
"max_tokens": 100,
"tools": [{"type": "function", "function": {"name": "get_weather", "description": "Get weather"}}],
},
"litellm_params": {"custom_llm_provider": "openai", "metadata": {}},
"standard_logging_object": payload,
"response_cost": 0.0015,
}
response_obj = {
"id": "chatcmpl-xyz789",
"model": "gpt-4o-2024-08-06",
"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hi there!"}}],
"usage": {"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20},
}
now = datetime.now(timezone.utc)
logger.log_pre_api_call("gpt-4o", kwargs["messages"], kwargs)
asyncio.run(
logger.async_log_success_event(
kwargs,
response_obj,
now,
now + timedelta(milliseconds=250),
)
)
finally:
detach(token)
# 1. No span, metrics, or events emitted by LiteLLM
assert span_exporter.get_finished_spans() == ()
assert _collected_metric_names(metric_reader) == []
assert log_exporter.get_finished_logs() == ()
# 2. Outer InferenceData is enriched via attributes and metric_attributes dicts
assert outer_data.attributes.get(GenAI.PROVIDER_NAME) == "openai"
assert outer_data.attributes.get(GenAI.OPERATION_NAME) == "chat"
assert outer_data.attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
assert outer_data.attributes.get(GenAI.REQUEST_TEMPERATURE) == 0.5
assert outer_data.attributes.get(GenAI.REQUEST_MAX_TOKENS) == 100
assert outer_data.attributes.get(GenAI.RESPONSE_ID) == "chatcmpl-xyz789"
assert list(outer_data.attributes.get(GenAI.RESPONSE_FINISH_REASONS) or []) == ["stop"]
assert outer_data.attributes.get(GenAI.USAGE_INPUT_TOKENS) == 12
assert outer_data.attributes.get(GenAI.USAGE_OUTPUT_TOKENS) == 8
assert outer_data.attributes.get(Server.ADDRESS) == "api.openai.com"
assert outer_data.attributes.get(Server.PORT) == 443
# Custom LiteLLM attributes published to outer_data.attributes
assert outer_data.attributes.get(LiteLLM.CALL_ID) == call_id
assert outer_data.attributes.get(LiteLLM.CACHE_HIT) is False
# Metric attributes published to outer_data.metric_attributes
assert outer_data.metric_attributes.get(GenAI.PROVIDER_NAME) == "openai"
assert outer_data.metric_attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
# Content capture attributes published into outer_data.attributes
assert outer_data.attributes.get(GenAI.INPUT_MESSAGES) is not None
assert outer_data.attributes.get(GenAI.OUTPUT_MESSAGES) is not None
def test_v2_suppressed_failure_enriches_outer_inference_context():
"""On a failed LLM call when suppressed, OpenTelemetryV2 suppresses its error span,
failure duration metric, and exception event, and publishes error details to outer_data."""
logger, span_exporter, metric_reader, log_exporter, *_ = _make_v2_harness()
outer_data = InferenceData()
token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data))
try:
call_id = "call-fail-suppressed-v2"
payload = _make_payload(
call_id=call_id,
model="gpt-4o",
custom_llm_provider="openai",
status="failure",
error_str="Rate limit exceeded",
error_information={
"error_code": "429",
"error_class": "RateLimitError",
"error_message": "Rate limit exceeded",
"llm_provider": "openai",
"traceback": "Traceback ...",
},
)
kwargs = {
"litellm_call_id": call_id,
"call_type": "completion",
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello"}],
"optional_params": {},
"litellm_params": {"custom_llm_provider": "openai", "metadata": {}},
"standard_logging_object": payload,
"exception": RuntimeError("Rate limit exceeded"),
}
now = datetime.now(timezone.utc)
logger.log_pre_api_call("gpt-4o", kwargs["messages"], kwargs)
asyncio.run(
logger.async_log_failure_event(
kwargs,
None,
now,
now + timedelta(milliseconds=100),
)
)
finally:
detach(token)
assert span_exporter.get_finished_spans() == ()
assert _collected_metric_names(metric_reader) == []
assert log_exporter.get_finished_logs() == ()
assert outer_data.attributes.get(Error.TYPE) == "RateLimitError"
assert outer_data.attributes.get(Error.MESSAGE) == "Rate limit exceeded"
def test_v2_suppressed_with_real_opentelemetry_util_genai_handler():
"""End-to-end test with opentelemetry.util.genai.handler.TelemetryHandler as the outer layer."""
util_genai = pytest.importorskip("opentelemetry.util.genai.handler")
(
logger,
span_exporter,
_metric_reader,
_log_exporter,
tracer_provider,
meter_provider,
logger_provider,
) = _make_v2_harness(capture_span_content=True)
handler = util_genai.TelemetryHandler(
tracer_provider=tracer_provider,
meter_provider=meter_provider,
logger_provider=logger_provider,
)
call_id = "call-e2e-outer-handler"
payload = _make_payload(
call_id=call_id,
model="claude-3-5-sonnet",
custom_llm_provider="anthropic",
prompt_tokens=15,
completion_tokens=9,
)
kwargs = {
"litellm_call_id": call_id,
"call_type": "completion",
"model": "claude-3-5-sonnet",
"messages": [{"role": "user", "content": "Hello Claude"}],
"optional_params": {"temperature": 0.2},
"litellm_params": {"custom_llm_provider": "anthropic", "metadata": {}},
"standard_logging_object": payload,
"response_cost": 0.003,
}
response_obj = {
"id": "msg-01",
"model": "claude-3-5-sonnet-20241022",
"choices": [{"finish_reason": "end_turn", "message": {"role": "assistant", "content": "Hello!"}}],
"usage": {"prompt_tokens": 15, "completion_tokens": 9, "total_tokens": 24},
}
now = datetime.now(timezone.utc)
with handler.inference(None, request_model="outer-model"):
logger.log_pre_api_call("claude-3-5-sonnet", kwargs["messages"], kwargs)
asyncio.run(
logger.async_log_success_event(
kwargs,
response_obj,
now,
now + timedelta(milliseconds=200),
)
)
spans = span_exporter.get_finished_spans()
# Exactly ONE client span is emitted (by the outer TelemetryHandler, enriched by LiteLLM)
assert len(spans) == 1
outer_span = spans[0]
attrs = dict(outer_span.attributes or {})
assert attrs.get(GenAI.PROVIDER_NAME) == "anthropic"
assert attrs.get(GenAI.USAGE_INPUT_TOKENS) == 15
assert attrs.get(GenAI.USAGE_OUTPUT_TOKENS) == 9
assert attrs.get(LiteLLM.CALL_ID) == call_id
def test_v2_embedding_context_suppression():
"""Embedding calls use CLIENT_EMBEDDING_CONTEXT_KEY and EmbeddingData."""
logger, span_exporter, metric_reader, *_ = _make_v2_harness()
outer_emb = EmbeddingData(request_model="text-embedding-3-small")
token = attach(set_value(CLIENT_EMBEDDING_CONTEXT_KEY, outer_emb))
try:
call_id = "call-emb-suppressed"
payload = _make_payload(
call_id=call_id,
call_type="embedding",
model="text-embedding-3-small",
custom_llm_provider="openai",
prompt_tokens=10,
completion_tokens=0,
)
kwargs = {
"litellm_call_id": call_id,
"call_type": "embedding",
"model": "text-embedding-3-small",
"messages": ["embed me"],
"optional_params": {"encoding_format": "float"},
"litellm_params": {"custom_llm_provider": "openai", "metadata": {}},
"standard_logging_object": payload,
}
now = datetime.now(timezone.utc)
logger.log_pre_api_call("text-embedding-3-small", kwargs["messages"], kwargs)
asyncio.run(
logger.async_log_success_event(
kwargs,
None,
now,
now + timedelta(milliseconds=100),
)
)
finally:
detach(token)
assert span_exporter.get_finished_spans() == ()
assert _collected_metric_names(metric_reader) == []
assert outer_emb.attributes.get(GenAI.PROVIDER_NAME) == "openai"
assert outer_emb.attributes.get(GenAI.REQUEST_MODEL) == "text-embedding-3-small"
assert outer_emb.attributes.get(GenAI.USAGE_INPUT_TOKENS) == 10