mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(otel): respect outer content capture opt-out and bound opentelemetry-util-genai
This commit is contained in:
parent
c0d506b557
commit
e8d1ade50e
4 changed files with 93 additions and 9 deletions
|
|
@ -20,6 +20,9 @@ from collections.abc import Mapping
|
|||
from typing import Final, cast
|
||||
|
||||
from opentelemetry.context import Context, get_value
|
||||
|
||||
# TODO: Import ContextData from the public opentelemetry.util.genai.invocation
|
||||
# module once exposed in the next opentelemetry-util-genai release.
|
||||
from opentelemetry.util.genai._invocation import _ContextData
|
||||
from opentelemetry.util.genai.invocation import (
|
||||
CLIENT_EMBEDDING_CONTEXT_KEY,
|
||||
|
|
@ -34,7 +37,7 @@ from opentelemetry.util.genai.invocation import (
|
|||
from opentelemetry.util.types import AttributeValue
|
||||
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData
|
||||
from litellm.integrations.otel.model.semconv import Error, GenAIOperation
|
||||
from litellm.integrations.otel.model.semconv import Error, GenAI, GenAIOperation
|
||||
|
||||
__all__ = [
|
||||
"CLIENT_EMBEDDING_CONTEXT_KEY",
|
||||
|
|
@ -66,6 +69,31 @@ _FETCH_RESPONSE_CALL_TYPES: Final[frozenset[str]] = frozenset(
|
|||
)
|
||||
|
||||
|
||||
_CONTENT_ATTRIBUTE_KEYS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
GenAI.INPUT_MESSAGES,
|
||||
GenAI.OUTPUT_MESSAGES,
|
||||
"langfuse.observation.input",
|
||||
"langfuse.observation.output",
|
||||
"input.value",
|
||||
"output.value",
|
||||
"gen_ai.prompt",
|
||||
"gen_ai.completion",
|
||||
}
|
||||
)
|
||||
|
||||
_CONTENT_ATTRIBUTE_PREFIXES: Final[tuple[str, ...]] = (
|
||||
"llm.input_messages.",
|
||||
"llm.output_messages.",
|
||||
"gen_ai.prompt.",
|
||||
"gen_ai.completion.",
|
||||
)
|
||||
|
||||
|
||||
def _is_content_attribute(key: str) -> bool:
|
||||
return key in _CONTENT_ATTRIBUTE_KEYS or key.startswith(_CONTENT_ATTRIBUTE_PREFIXES)
|
||||
|
||||
|
||||
def context_spec_for_operation(
|
||||
operation: GenAIOperation | str | None,
|
||||
call_type: str | None = None,
|
||||
|
|
@ -112,7 +140,10 @@ def publish_to_genai_context(
|
|||
metric_attrs: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
"""Publish LiteLLM's mapped span and metric attributes into an outer ``ctx_data``."""
|
||||
allow_content: Final = bool(ctx_data.capture_content)
|
||||
for key, val in span_attrs.items():
|
||||
if not allow_content and _is_content_attribute(key):
|
||||
continue
|
||||
if val is not None and isinstance(val, (str, bool, int, float, list, tuple)):
|
||||
ctx_data.attributes[key] = cast(AttributeValue, val)
|
||||
|
||||
|
|
|
|||
|
|
@ -643,9 +643,12 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
|
||||
carrier.span.end(end_time=to_ns(end_time))
|
||||
return None
|
||||
capture_content: Final = self._capture_span_content() and (
|
||||
outer_ctx_data is None or bool(getattr(outer_ctx_data, "capture_content", False))
|
||||
)
|
||||
data: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
payload,
|
||||
capture_content=self._capture_span_content(),
|
||||
capture_content=capture_content,
|
||||
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
|
||||
completion_start_seconds=call.completion_start_seconds,
|
||||
request_route=request_root_http_route(),
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ proxy-runtime = [
|
|||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"opentelemetry-util-genai>=1.4b0",
|
||||
"opentelemetry-util-genai>=1.4b0,<2.0",
|
||||
"ddtrace>=4.8.2,<5.0",
|
||||
"sentry-sdk>=2.21.0,<3.0",
|
||||
"mangum>=0.17.0,<1.0",
|
||||
|
|
@ -232,7 +232,7 @@ dev = [
|
|||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"opentelemetry-util-genai>=1.4b0",
|
||||
"opentelemetry-util-genai>=1.4b0,<2.0",
|
||||
"langfuse>=4.7,<5.0",
|
||||
"fastapi-offline==1.7.6",
|
||||
"fakeredis==2.34.1",
|
||||
|
|
@ -265,7 +265,7 @@ proxy-dev = [
|
|||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"opentelemetry-util-genai>=1.4b0",
|
||||
"opentelemetry-util-genai>=1.4b0,<2.0",
|
||||
"azure-identity==1.25.2",
|
||||
"a2a-sdk==1.1.0",
|
||||
"backports-zstd==1.8.0; python_version < '3.14'",
|
||||
|
|
|
|||
|
|
@ -167,7 +167,10 @@ def test_v2_suppressed_when_outer_inference_context_active():
|
|||
suppresses its own client span and metrics and enriches the outer InferenceData."""
|
||||
logger, span_exporter, metric_reader, log_exporter, *_ = _make_v2_harness(capture_span_content=True)
|
||||
|
||||
outer_data = InferenceData(request_model="langchain-alias")
|
||||
outer_data = InferenceData(
|
||||
request_model="langchain-alias",
|
||||
capture_content=True,
|
||||
)
|
||||
token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data))
|
||||
try:
|
||||
call_id = "call-suppressed-v2"
|
||||
|
|
@ -232,11 +235,60 @@ def test_v2_suppressed_when_outer_inference_context_active():
|
|||
assert outer_data.metric_attributes.get(GenAI.PROVIDER_NAME) == "openai"
|
||||
assert outer_data.metric_attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
|
||||
|
||||
# Content capture attributes published into outer_data.attributes
|
||||
# Content capture attributes published into outer_data.attributes when outer capture_content=True
|
||||
assert outer_data.attributes.get(GenAI.INPUT_MESSAGES) is not None
|
||||
assert outer_data.attributes.get(GenAI.OUTPUT_MESSAGES) is not None
|
||||
|
||||
|
||||
def test_v2_suppressed_respects_outer_content_capture_opt_out():
|
||||
"""When LiteLLM has capture_span_content=True but the outer InferenceData has
|
||||
capture_content=False (default), prompt and response content attributes must not be copied."""
|
||||
logger, span_exporter, *_ = _make_v2_harness(capture_span_content=True)
|
||||
|
||||
outer_data = InferenceData(request_model="langchain-alias", capture_content=False)
|
||||
token = attach(set_value(CLIENT_INFERENCE_CONTEXT_KEY, outer_data))
|
||||
try:
|
||||
call_id = "call-suppressed-no-content"
|
||||
payload = _make_payload(call_id=call_id, model="gpt-4o", custom_llm_provider="openai")
|
||||
kwargs = {
|
||||
"litellm_call_id": call_id,
|
||||
"call_type": "completion",
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "Secret user prompt"}],
|
||||
"optional_params": {"temperature": 0.5},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "metadata": {}},
|
||||
"standard_logging_object": payload,
|
||||
}
|
||||
response_obj = {
|
||||
"id": "chatcmpl-xyz789",
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Secret reply"}}],
|
||||
"usage": {"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20},
|
||||
}
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
logger.log_pre_api_call("gpt-4o", kwargs["messages"], kwargs)
|
||||
asyncio.run(
|
||||
logger.async_log_success_event(
|
||||
kwargs,
|
||||
response_obj,
|
||||
now,
|
||||
now + timedelta(milliseconds=250),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
detach(token)
|
||||
|
||||
assert span_exporter.get_finished_spans() == ()
|
||||
assert outer_data.attributes.get(GenAI.PROVIDER_NAME) == "openai"
|
||||
assert GenAI.INPUT_MESSAGES not in outer_data.attributes
|
||||
assert GenAI.OUTPUT_MESSAGES not in outer_data.attributes
|
||||
assert "input.value" not in outer_data.attributes
|
||||
assert "output.value" not in outer_data.attributes
|
||||
assert "langfuse.observation.input" not in outer_data.attributes
|
||||
assert "langfuse.observation.output" not in outer_data.attributes
|
||||
|
||||
|
||||
def test_v2_suppressed_failure_enriches_outer_inference_context():
|
||||
"""On a failed LLM call when suppressed, OpenTelemetryV2 suppresses its error span,
|
||||
failure duration metric, and exception event, and publishes error details to outer_data."""
|
||||
|
|
@ -403,5 +455,3 @@ def test_v2_embedding_context_suppression():
|
|||
assert outer_emb.attributes.get(GenAI.PROVIDER_NAME) == "openai"
|
||||
assert outer_emb.attributes.get(GenAI.REQUEST_MODEL) == "text-embedding-3-small"
|
||||
assert outer_emb.attributes.get(GenAI.USAGE_INPUT_TOKENS) == 10
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue