fix(otel): read the attribute budget from the span's own provider limits so routed tracers fit correctly

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/integrations/otel/emitter.py

@@ -7,7 +7,7 @@ from typing import Final
 
 from opentelemetry.context import Context
 from opentelemetry.sdk.trace import ReadableSpan, SpanLimits
-from opentelemetry.sdk.trace import Tracer as SdkTracer
+from opentelemetry.sdk.trace import Span as SdkSpan
 from opentelemetry.trace import Link, Span, Tracer
 from opentelemetry.trace.status import Status, StatusCode
 
@@ -84,11 +84,20 @@ def error_attributes(error: SpanError) -> Mapping[str, AttrValue]:
     return MappingProxyType({key: value for key, value in pairs if value})
 
 
-def span_attribute_limit(tracer: Tracer) -> int | None:
-    """The attribute count limit spans started by ``tracer`` are built with, ``None`` when unbounded."""
-    if not isinstance(tracer, SdkTracer):
+def span_attribute_limit(span: Span) -> int | None:
+    """The attribute count limit ``span`` was built with, ``None`` when unbounded."""
+    if not isinstance(span, SdkSpan):
         return SpanLimits().max_span_attributes
-    return tracer._span_limits.max_span_attributes  # pyright: ignore[reportPrivateUsage]  # SDK has no public getter
+    return span._limits.max_span_attributes  # pyright: ignore[reportPrivateUsage]  # SDK has no public getter
+
+
+def attribute_budget(span: Span, reserved: int) -> int | None:
+    """How many mapped attributes fit on ``span`` next to what it already carries and ``reserved`` more."""
+    limit: Final = span_attribute_limit(span)
+    if limit is None:
+        return None
+    on_span: Final = len(span.attributes or ()) if isinstance(span, ReadableSpan) else 0
+    return limit - on_span - reserved
 
 
 def stamp_error(
@@ -138,7 +147,6 @@ class SpanEmitter:
         self._tracer = tracer
         self._config = config
         self._event_recorder = event_recorder
-        self._span_attribute_limit: int | None = span_attribute_limit(tracer)
         # The mapper chain is the sole source of span attributes. When not
         # passed in, resolve it from the config so there's one source of truth.
         self._mappers: list[AttributeMapper] = (
@@ -276,7 +284,7 @@ class SpanEmitter:
         )
         stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
         reserved: Final = len(stamped_later.keys() - mapped.keys())
-        for key, value in fit_indexed_messages(mapped, self._attribute_budget(span, reserved)).items():
+        for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved)).items():
             span.set_attribute(key, value)
         if error:
             stamped: Final = stamp_error(span, error)
@@ -294,10 +302,3 @@ class SpanEmitter:
         # span-level health signal litellm doesn't actually evaluate. Only a
         # genuine error sets a status.
         span.end(end_time=end_time_ns)
-
-    def _attribute_budget(self, span: Span, reserved: int) -> int | None:
-        """How many mapped attributes fit on ``span`` next to what it already carries and ``reserved`` more."""
-        if self._span_attribute_limit is None:
-            return None
-        on_span: Final = len(span.attributes or ()) if isinstance(span, ReadableSpan) else 0
-        return self._span_attribute_limit - on_span - reserved

tests/test_litellm/integrations/otel/test_otel_v2_emitter.py

@@ -10,7 +10,7 @@ pytest.importorskip("opentelemetry")
 from opentelemetry.sdk.trace import SpanLimits, TracerProvider  # noqa: E402
 from opentelemetry.sdk.trace.export import SimpleSpanProcessor  # noqa: E402
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter  # noqa: E402
-from opentelemetry.trace import NoOpTracer, SpanKind  # noqa: E402
+from opentelemetry.trace import INVALID_SPAN, SpanKind  # noqa: E402
 from opentelemetry.trace.status import StatusCode  # noqa: E402
 
 from litellm.integrations.otel import (  # noqa: E402
@@ -662,9 +662,29 @@ def test_indexed_messages_follow_the_providers_own_span_limits(monkeypatch):
     assert _indexed_messages(unbounded.attributes, "llm.input_messages") == list(range(60))
 
 
-def test_span_attribute_limit_falls_back_to_the_environment_for_tracers_outside_the_sdk(monkeypatch):
+def test_indexed_messages_follow_the_span_limits_of_a_per_request_tracer_override(monkeypatch):
+    """A routed ``tracer`` builds the span, so its provider's limits set the budget, not the bound tracer's."""
+    monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", "1000")
+    cfg = OpenTelemetryV2Config(
+        exporter="in_memory", mapper_names=["genai", "openinference"], capture_message_content="span_only"
+    )
+    bound_provider, _ = _provider_with_limits(SpanLimits(max_span_attributes=1000))
+    routed_provider, routed_exporter = _provider_with_limits(SpanLimits(max_span_attributes=40))
+    engine = SpanEmitter(providers.get_tracer(bound_provider, "litellm-test"), cfg)
+    engine.emit(
+        SpanRole.LLM_CALL,
+        LLMCallSpanData.from_standard_logging_payload(_conversation_payload(60), capture_content=True),
+        tracer=providers.get_tracer(routed_provider, "litellm-routed"),
+    )
+    (span,) = routed_exporter.get_finished_spans()
+    _assert_core_intact(span)
+    assert 39 <= len(span.attributes) <= 40
+    assert span.attributes["llm.output_messages.0.message.content"] == "reply 0"
+
+
+def test_span_attribute_limit_falls_back_to_the_environment_for_spans_outside_the_sdk(monkeypatch):
     monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", "48")
-    assert span_attribute_limit(NoOpTracer()) == 48
+    assert span_attribute_limit(INVALID_SPAN) == 48
 
 
 def test_fully_populated_span_with_every_vocabulary_stays_within_the_attribute_limit():