fix(langfuse_otel): keep the derived trace fields from pushing other attributes off a full span

The derived trace name, tags, input and output were written after the OpenInference attributes with no room
check, so a span near the SDK's attribute limit (about 34 tools at the default 128) evicted keys main
exported, such as openinference.span.kind, metadata and llm.model_name, and raised the dropped count.
They are now written last and only into the room the limit leaves next to the error attributes a failed
call adds, so a full span exports what it would without them.
This commit is contained in:
yucheng-berri 2026-10-20 19:43:36 -07:00
parent 6d4f6aaa68
commit 0ba3a15084
3 changed files with 170 additions and 24 deletions

View file

@ -11,12 +11,18 @@ from opentelemetry import trace
import litellm
from litellm._logging import verbose_logger
from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations._types.open_inference import ErrorAttributes
from litellm.integrations.arize import _utils
from litellm.integrations.langfuse.langfuse_otel_attributes import (
LangfuseLLMObsOTELAttributes,
serialize_langfuse_datetime,
)
from litellm.integrations.opentelemetry import LITELLM_PROXY_REQUEST_SPAN_NAME, OpenTelemetry, OpenTelemetryConfig
from litellm.integrations.opentelemetry import (
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
LITELLM_PROXY_REQUEST_SPAN_NAME,
OpenTelemetry,
OpenTelemetryConfig,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.secret_managers.main import str_to_bool
@ -37,6 +43,17 @@ LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel"
LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel"
LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version"
LANGFUSE_INGESTION_VERSION: Final = "4"
# The attributes ``OpenTelemetry._record_exception_on_span`` adds to a failed call's span after these.
_ERROR_ATTRIBUTES: Final = frozenset(
{
ErrorAttributes.ERROR_CODE,
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
ErrorAttributes.ERROR_TYPE,
ErrorAttributes.ERROR_MESSAGE,
ErrorAttributes.ERROR_LLM_PROVIDER,
ErrorAttributes.ERROR_STACK_TRACE,
}
)
@runtime_checkable
@ -283,27 +300,9 @@ class LangfuseOtelLogger(OpenTelemetry):
joins_existing_trace: Final = metadata.get("existing_trace_id") is not None or not _starts_its_own_trace(
span, kwargs
)
if metadata.get("trace_name") is None and not joins_existing_trace:
safe_set_attribute(
span,
LangfuseSpanAttributes.TRACE_NAME.value,
f"litellm-{kwargs.get('call_type') or 'completion'}",
)
if not joins_existing_trace:
tags: Final = LangfuseOtelLogger._trace_tags(kwargs, metadata)
if tags:
safe_set_attribute(span, LangfuseSpanAttributes.TAGS.value, json.dumps(list(tags)))
input_json: Final = safe_dumps(kwargs.get("messages")) if kwargs.get("messages") else None
if input_json is not None:
safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_INPUT.value, input_json)
if not joins_existing_trace:
safe_set_attribute(
span,
LangfuseSpanAttributes.TRACE_INPUT.value,
REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_input")) else input_json,
)
completion_start_time: Final = kwargs.get("completion_start_time")
if isinstance(completion_start_time, datetime):
@ -316,12 +315,63 @@ class LangfuseOtelLogger(OpenTelemetry):
output_json: Final = LangfuseOtelLogger._observation_output(response_obj)
if output_json is not None:
safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, output_json)
if not joins_existing_trace:
safe_set_attribute(
span,
LangfuseSpanAttributes.TRACE_OUTPUT.value,
REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_output")) else output_json,
if not joins_existing_trace:
LangfuseOtelLogger._set_trace_fields(
span=span, kwargs=kwargs, metadata=metadata, input_json=input_json, output_json=output_json
)
@staticmethod
def _set_trace_fields(
span: Span,
kwargs: Mapping[str, Any],
metadata: Mapping[str, object],
input_json: str | None,
output_json: str | None,
) -> None:
"""Set the derived trace name, tags, input and output into the room ``span``'s attribute limit leaves next
to the error attributes a failed call adds after them, so they never evict or drop another attribute."""
from litellm.integrations.otel.emitter import carried_keys, span_attribute_limit
tags: Final = LangfuseOtelLogger._trace_tags(kwargs, metadata)
fields: Final = (
*(
((LangfuseSpanAttributes.TRACE_NAME.value, f"litellm-{kwargs.get('call_type') or 'completion'}"),)
if metadata.get("trace_name") is None
else ()
),
*(((LangfuseSpanAttributes.TAGS.value, json.dumps(list(tags))),) if tags else ()),
*(
(
(
LangfuseSpanAttributes.TRACE_INPUT.value,
REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_input")) else input_json,
),
)
if input_json is not None
else ()
),
*(
(
(
LangfuseSpanAttributes.TRACE_OUTPUT.value,
REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_output")) else output_json,
),
)
if output_json is not None
else ()
),
)
limit: Final = span_attribute_limit(span)
carried: Final = carried_keys(span)
new_keys: Final = tuple(key for key, _value in fields if key not in carried)
room: Final = (
len(new_keys) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0)
)
written: Final = carried | frozenset(new_keys[:room])
for key, value in fields:
if key in written:
_utils.safe_set_attribute(span, key, value)
@staticmethod
def _get_langfuse_otel_host() -> str | None:

View file

@ -38,6 +38,7 @@ class SpanRecord:
trace_id: str
parent_span_id: str
attributes: Mapping[str, object]
dropped_attributes: int = 0
def tags(self) -> tuple[object, ...]:
"""The trace tags Langfuse ingestion reads from the raw attribute."""
@ -71,6 +72,7 @@ def _span_records(body: bytes) -> tuple[SpanRecord, ...]:
trace_id=span.trace_id.hex(),
parent_span_id=span.parent_span_id.hex(),
attributes={pair.key: _attribute_value(pair.value) for pair in span.attributes},
dropped_attributes=span.dropped_attributes_count,
)
for span in _exported_spans(body)
)
@ -613,3 +615,44 @@ def test_destination_outage_mid_burst_lands_every_span_once_with_derived_fields(
assert {identity: len(records) for identity, records in spans.items()} == dict.fromkeys(expected, 1)
for identity, records in spans.items():
_assert_derived(records[0], SURFACE_CALL_TYPES[expected[identity]])
def _tool(index: int) -> Mapping[str, object]:
return {
"type": "function",
"function": {
"name": f"lookup_{index}",
"description": f"Looks up record {index}",
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}},
},
}
def _chat_with_tools(rig: Rig, content: str, tools: int, headers: Mapping[str, str] | None = None) -> SpanRecord:
response: Final = rig.proxy.request(
"POST",
"/v1/chat/completions",
{
"model": rig.openai_model,
"messages": [{"role": "user", "content": content}],
"tools": [_tool(index) for index in range(tools)],
},
key=rig.key,
headers=headers,
)
assert response.status_code == (500 if content.startswith("fail-") else 200), response.text
return rig.sink.one(content)
@pytest.mark.parametrize(("tools", "prefix"), product((30, 33, 34, 35, 36, 40), ("", "fail-")))
def test_trace_fields_never_evict_or_drop_another_attribute_of_a_full_span(v1: Rig, tools: int, prefix: str) -> None:
run: Final = uuid.uuid4().hex
derived: Final = _chat_with_tools(v1, f"{prefix}lit8281-full-{tools}-{run}", tools)
joined: Final = _chat_with_tools(
v1,
f"{prefix}lit8281-full-joined-{tools}-{run}",
tools,
headers={"traceparent": f"00-{uuid.uuid4().hex}-{uuid.uuid4().hex[:16]}-01"},
)
assert set(joined.attributes) <= set(derived.attributes), set(joined.attributes) - set(derived.attributes)
assert derived.dropped_attributes == joined.dropped_attributes, (derived.dropped_attributes, joined.attributes)

View file

@ -2,6 +2,7 @@ import base64
import json
import os
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Final
from unittest.mock import MagicMock, patch
@ -1385,6 +1386,58 @@ class TestDerivedTraceFieldsOnlyOnTracesLitellmStarts:
}
ERROR_KEYS: Final = (
"error.code",
"http.response.status_code",
"error.type",
"error.message",
"error.llm_provider",
"error.stack_trace",
)
SPAN_ATTRIBUTE_LIMIT: Final = 128
@dataclass(frozen=True, slots=True)
class ExportedSpan:
keys: frozenset[str]
dropped: int
def _exported_full_span(filler: int, *, joins_caller: bool, failed: bool) -> ExportedSpan:
"""A span already holding ``filler`` attributes, after langfuse_otel stamps a chat call and, when ``failed``, the
error attributes a failed call adds."""
span: Final = _TRACER.start_span(
"litellm_request",
context=trace.set_span_in_context(trace.NonRecordingSpan(CALLER)) if joins_caller else Context(),
)
span.set_attributes({f"main.{index}": index for index in range(filler)})
LangfuseOtelLogger._set_langfuse_specific_attributes(
span, {**CHAT_KWARGS, "litellm_params": {"metadata": {}}}, _chat_response("ok")
)
span.set_attributes(dict.fromkeys(ERROR_KEYS, "scripted failure") if failed else {})
span.end()
return ExportedSpan(frozenset(getattr(span, "attributes", None) or {}), getattr(span, "dropped_attributes", 0))
class TestDerivedTraceFieldsAtTheSpanAttributeLimit:
"""The derived trace fields take only room a span has to spare, so a full span exports what it would without
them."""
@pytest.mark.parametrize("failed", [False, True])
@pytest.mark.parametrize("filler", range(SPAN_ATTRIBUTE_LIMIT - 30, SPAN_ATTRIBUTE_LIMIT + 1))
def test_trace_fields_never_evict_or_drop_another_attribute(self, filler: int, failed: bool) -> None:
derived: Final = _exported_full_span(filler, joins_caller=False, failed=failed)
without_trace_fields: Final = _exported_full_span(filler, joins_caller=True, failed=failed)
assert without_trace_fields.keys <= derived.keys
assert derived.dropped == without_trace_fields.dropped
def test_a_span_with_room_to_spare_carries_every_trace_field(self) -> None:
derived: Final = _exported_full_span(SPAN_ATTRIBUTE_LIMIT - 30, joins_caller=False, failed=False)
assert {"langfuse.trace.name", "langfuse.trace.input", "langfuse.trace.output"} <= derived.keys
LANGFUSE_ENV_ONLY: Final = {
key: value
for key, value in os.environ.items()