feat(otel v2): per-destination otel_internal_spans to hold back the proxy's SERVICE and DB_CALL spans from tenant exporters

A key or team that points langfuse_otel, arize, weave_otel or newrelic at its own account can now set
otel_internal_spans to exclude, and the TenantFanOutSpanProcessor stops forwarding the proxy's own auth,
Redis, Postgres and spend-write spans to that destination. The request root and the tenant's model, guardrail
and MCP spans still go, and the operator's exporter still receives the full tree. The default stays include,
with litellm_settings.otel_tenant_internal_spans (or LITELLM_OTEL_TENANT_INTERNAL_SPANS) as the system-wide
default for destinations that name no value. The team and key Logging Settings forms render the field as a
select; the global Add Callback form does not, since the operator exporter cannot honour it.

Resolves LIT-8245

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/__init__.py

@@ -328,6 +328,9 @@ provider_url_destination_allowed_hosts: List[str] = []
 #: "override" (default) or "additive": whether a key or team destination replaces
 #: the operator's exporter for that backend or exports alongside it.
 otel_tenant_destination_mode: str | None = None
+#: "include" (default) or "exclude": whether a key or team destination receives the
+#: proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres, spend writes).
+otel_tenant_internal_spans: str | None = None
 ssl_ecdh_curve: Optional[str] = None  # Set to 'X25519' to disable PQC and improve performance
 disable_streaming_logging: bool = False
 disable_token_counter: bool = False

litellm/integrations/callback_configs.json

@@ -16,6 +16,13 @@
         "ui_name": "Space ID",
         "description": "Arize Space ID to identify your workspace",
         "required": true
+      },
+      "otel_internal_spans": {
+        "type": "select",
+        "ui_name": "Internal Spans",
+        "description": "include sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, exclude holds them back",
+        "options": ["include", "exclude"],
+        "required": false
       }
     },
     "description": "Arize Logging Integration"
@@ -266,6 +273,13 @@
         "description": "full sends the whole request trace, llm_only sends just the model-call spans",
         "options": ["full", "llm_only"],
         "required": false
+      },
+      "otel_internal_spans": {
+        "type": "select",
+        "ui_name": "Internal Spans",
+        "description": "include sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, exclude holds them back",
+        "options": ["include", "exclude"],
+        "required": false
       }
     },
     "description": "Langfuse v3 OTEL Logging Integration"
@@ -326,6 +340,13 @@
         "ui_name": "New Relic Region (us or eu)",
         "description": "Data center region for this team's account. Defaults to us.",
         "required": false
+      },
+      "otel_internal_spans": {
+        "type": "select",
+        "ui_name": "Internal Spans",
+        "description": "include sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, exclude holds them back",
+        "options": ["include", "exclude"],
+        "required": false
       }
     },
     "description": "New Relic AI Monitoring Integration"

litellm/integrations/otel/model/destination.py

@@ -10,7 +10,7 @@ from urllib.parse import quote
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from litellm.types.utils import OtelSpanScope
+from litellm.types.utils import OtelInternalSpans, OtelSpanScope
 
 
 class OtelDestination(BaseModel):
@@ -31,6 +31,13 @@ class OtelDestination(BaseModel):
         default="full",
         description="``llm_only`` keeps just the model-call spans; the rest of the request tree is not forwarded.",
     )
+    internal_spans: OtelInternalSpans = Field(
+        default="include",
+        description=(
+            "``exclude`` holds back the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres, "
+            "spend writes); the request root and the tenant's own model, guardrail and MCP spans still go."
+        ),
+    )
 
     def header_string(self) -> str:
         """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects.
@@ -45,9 +52,9 @@ class OtelDestination(BaseModel):
     def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]:
         """Identity for processor reuse, so one destination means one exporter.
 
-        ``span_scope`` is left out on purpose: the scope decides which spans reach the
-        processor, not how the processor exports them, so a full and an ``llm_only``
-        view of the same account share one exporter.
+        ``span_scope`` and ``internal_spans`` are left out on purpose: they decide which
+        spans reach the processor, not how the processor exports them, so a full and an
+        ``llm_only`` view of the same account share one exporter.
         """
         return (
             self.endpoint,

litellm/integrations/otel/plumbing/context.py

@@ -21,6 +21,7 @@ from opentelemetry.trace.propagation.tracecontext import (
 )
 
 from litellm.integrations.otel.model.semconv import HTTP
+from litellm.types.utils import OtelInternalSpans
 
 if TYPE_CHECKING:
     from litellm.integrations.otel.model.destination import OtelDestination
@@ -400,6 +401,24 @@ def tenant_destinations_are_additive() -> bool:
     return isinstance(configured, str) and configured.strip().lower() == ADDITIVE_DESTINATION_MODE
 
 
+EXCLUDE_INTERNAL_SPANS: Final = "exclude"
+OTEL_TENANT_INTERNAL_SPANS_ENV: Final = "LITELLM_OTEL_TENANT_INTERNAL_SPANS"
+
+
+def tenant_internal_spans_default() -> OtelInternalSpans:
+    """The ``internal_spans`` a tenant destination gets when its own callback vars name none.
+
+    Include is the default, so a destination keeps the whole request tree unless the
+    operator or the team asks for the proxy's own SERVICE and DB_CALL spans to stay home.
+    """
+    import litellm
+
+    configured: Final = litellm.otel_tenant_internal_spans or os.environ.get(OTEL_TENANT_INTERNAL_SPANS_ENV)
+    if isinstance(configured, str) and configured.strip().lower() == EXCLUDE_INTERNAL_SPANS:
+        return "exclude"
+    return "include"
+
+
 def destination_backends() -> frozenset[str]:
     """Backends this request resolved a tenant destination for.
 

litellm/integrations/otel/plumbing/providers.py

@@ -440,6 +440,22 @@ def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool:
     return scope == "full" or is_llm_call_span(span)
 
 
+def _is_internal_span(span: ReadableSpan) -> bool:
+    """Whether ``span`` is the proxy's own work rather than the request or the tenant's call.
+
+    The request root is the SERVER span; the model call, the MCP call and the guardrail
+    each carry one of the tenant-owned keys. Everything else the proxy emits inside a
+    request is a SERVICE or DB_CALL span: auth, Redis, Postgres, spend writes.
+    """
+    return span.kind is not SpanKind.SERVER and not _is_tenant_owned_span(span.attributes or _NO_ATTRIBUTES)
+
+
+def _forwarded(span: ReadableSpan, destination: "OtelDestination") -> bool:
+    if not _in_scope(span, destination.span_scope):
+        return False
+    return destination.internal_spans == "include" or not _is_internal_span(span)
+
+
 def _scoped(span: ReadableSpan, scope: "OtelSpanScope") -> ReadableSpan:
     """Under ``llm_only`` the model call is the only span the exporter gets, so it goes out as the
     trace's root (its parent is the request span that is held back) and, unless the caller named the
@@ -570,9 +586,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
     def on_end(self, span: ReadableSpan) -> None:
         suppressed: Final = suppressed_backends()
         for destination in request_destinations():
-            if self._operator_already_writes(span, destination, suppressed) or not _in_scope(
-                span, destination.span_scope
-            ):
+            if self._operator_already_writes(span, destination, suppressed) or not _forwarded(span, destination):
                 continue
             processor = self._acquire(destination)  # rebind-ok: loop variable; pyright forbids Final in a loop
             if processor is None:

litellm/integrations/otel/presets/destinations.py

@@ -14,8 +14,9 @@ from typing import Final
 import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.otel.model.destination import OtelDestination
+from litellm.integrations.otel.plumbing.context import tenant_internal_spans_default
 from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host
-from litellm.types.utils import OtelSpanScope, StandardCallbackDynamicParams
+from litellm.types.utils import OtelInternalSpans, OtelSpanScope, StandardCallbackDynamicParams
 
 #: An endpoint plus the OTLP transport to reach it with, or ``None`` when the backend
 #: names no destination. The transport is ``None`` where the backend has only one.
@@ -117,6 +118,11 @@ def _span_scope(callback_name: str, params: StandardCallbackDynamicParams) -> Ot
     return params.get("langfuse_span_scope") or "full"
 
 
+def _internal_spans(params: StandardCallbackDynamicParams) -> OtelInternalSpans:
+    configured: Final = params.get("otel_internal_spans")
+    return tenant_internal_spans_default() if configured is None else configured
+
+
 def destination_capable_backends() -> frozenset[str]:
     """Backends a key or team can point at its own account."""
     from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK
@@ -156,4 +162,5 @@ def destination_for(
         callback_name=callback_name,
         protocol=protocol,
         span_scope=_span_scope(callback_name, params),
+        internal_spans=_internal_spans(params),
     )

litellm/litellm_core_utils/initialize_dynamic_callback_params.py

@@ -4,7 +4,12 @@ from contextlib import contextmanager
 from contextvars import ContextVar
 from typing import Any, Final
 
-from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
+from litellm.types.utils import (
+    OTEL_INTERNAL_SPAN_CHOICES,
+    OTEL_SPAN_SCOPES,
+    TRUSTED_CALLBACK_VARS_FIELD,
+    StandardCallbackDynamicParams,
+)
 
 _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
 _inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
@@ -81,6 +86,11 @@ def validate_langfuse_span_scope_value(value: str) -> None:
         raise ValueError(f"Invalid langfuse_span_scope {value!r}: must be one of {sorted(OTEL_SPAN_SCOPES)}")
 
 
+def validate_otel_internal_spans_value(value: str) -> None:
+    if value not in OTEL_INTERNAL_SPAN_CHOICES:
+        raise ValueError(f"Invalid otel_internal_spans {value!r}: must be one of {sorted(OTEL_INTERNAL_SPAN_CHOICES)}")
+
+
 # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
 _supported_callback_params: Final[tuple[str, ...]] = (
     "langfuse_public_key",

litellm/proxy/_types.py

@@ -26,6 +26,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     validate_langfuse_environment_value,
     validate_langfuse_span_scope_value,
     validate_no_callback_env_reference,
+    validate_otel_internal_spans_value,
 )
 from litellm.types.agents import AgentCaller
 from litellm.types.integrations.compression_interception import (
@@ -2258,6 +2259,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase):
                 validate_langfuse_environment_value(callback_vars[key])
             if key == "langfuse_span_scope":
                 validate_langfuse_span_scope_value(callback_vars[key])
+            if key == "otel_internal_spans":
+                validate_otel_internal_spans_value(callback_vars[key])
         return values
 
 

litellm/proxy/common_utils/callback_config_validation.py

@@ -14,6 +14,7 @@ _NEWRELIC_CALLBACK: Final = "newrelic"
 _NEWRELIC_VAR_PREFIX: Final = "newrelic_"
 _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel"
 _LANGFUSE_SPAN_SCOPE_VAR: Final = "langfuse_span_scope"
+_OTEL_INTERNAL_SPANS_VAR: Final = "otel_internal_spans"
 _ARIZE_CALLBACK: Final = "arize"
 _ARIZE_SAMPLING_RATE_VARS: Final[frozenset[str]] = frozenset(
     {"arize_success_sampling_rate", "arize_error_sampling_rate"}
@@ -31,6 +32,9 @@ def callback_config_error(callback_name: str | None, callback_vars: Mapping[str,
     )
     if langfuse_error is not None:
         return langfuse_error
+    internal_spans_error: Final = _otel_internal_spans_error(callback_name, callback_vars)
+    if internal_spans_error is not None:
+        return internal_spans_error
     if callback_name != _NEWRELIC_CALLBACK:
         return None
     return _newrelic_config_error(callback_vars)
@@ -75,6 +79,27 @@ def _langfuse_span_scope_error(callback_name: str | None, callback_vars: Mapping
     return None
 
 
+def _otel_internal_spans_error(callback_name: str | None, callback_vars: Mapping[str, str]) -> str | None:
+    value: Final = callback_vars.get(_OTEL_INTERNAL_SPANS_VAR)
+    if value is None:
+        return None
+    from litellm.integrations.otel.presets.destinations import destination_capable_backends
+    from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
+        validate_otel_internal_spans_value,
+    )
+
+    if callback_name not in destination_capable_backends():
+        return (
+            f"{_OTEL_INTERNAL_SPANS_VAR} applies to the OTEL destination callbacks only "
+            f"({', '.join(sorted(destination_capable_backends()))}), not {callback_name!r}"
+        )
+    try:
+        validate_otel_internal_spans_value(value)
+    except ValueError as e:
+        return str(e)
+    return None
+
+
 # Which credential family a dynamic variable belongs to. The families are the
 # integrations that share one account: every langfuse_* variable configures the
 # same Langfuse project whether it rides the classic callback or the OTel one,
@@ -169,15 +194,32 @@ def conflicting_span_scope_error(
     callback_vars: Mapping[str, str] | None,
     stored_vars_by_entry: Sequence[Mapping[str, str]],
 ) -> str | None:
-    incoming: Final = None if callback_vars is None else callback_vars.get(_LANGFUSE_SPAN_SCOPE_VAR)
+    return _conflicting_var_error(_LANGFUSE_SPAN_SCOPE_VAR, callback_vars, stored_vars_by_entry)
+
+
+def conflicting_internal_spans_error(
+    callback_vars: Mapping[str, str] | None,
+    stored_vars_by_entry: Sequence[Mapping[str, str]],
+) -> str | None:
+    """``stored_vars_by_entry`` must hold only the entries of the same callback: the
+    request merges the var per backend, so two backends may legitimately disagree."""
+    return _conflicting_var_error(_OTEL_INTERNAL_SPANS_VAR, callback_vars, stored_vars_by_entry)
+
+
+def _conflicting_var_error(
+    var: str,
+    callback_vars: Mapping[str, str] | None,
+    stored_vars_by_entry: Sequence[Mapping[str, str]],
+) -> str | None:
+    incoming: Final = None if callback_vars is None else callback_vars.get(var)
     if incoming is None:
         return None
     return next(
         (
-            f"{_LANGFUSE_SPAN_SCOPE_VAR} is already set to {stored!r} by another callback entry. "
-            f"Every entry shares one scope: remove that entry or send the same value."
+            f"{var} is already set to {stored!r} by another callback entry. "
+            f"Every entry shares one value: remove that entry or send the same value."
             for entry in stored_vars_by_entry
-            if (stored := entry.get(_LANGFUSE_SPAN_SCOPE_VAR)) not in (None, incoming)
+            if (stored := entry.get(var)) not in (None, incoming)
         ),
         None,
     )
@@ -191,12 +233,20 @@ def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str
     if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)):
         return None
     entry_vars: Final = tuple(_entry_callback_vars(entry) for entry in entries)
+    entry_names: Final = tuple(_entry_callback_name(entry) for entry in entries)
     return next(
         (
             error
             for error in (
                 *(_logging_entry_error(entry) for entry in entries),
                 *(conflicting_span_scope_error(entry_vars[i], entry_vars[:i]) for i in range(len(entry_vars))),
+                *(
+                    conflicting_internal_spans_error(
+                        entry_vars[i],
+                        tuple(vars_ for vars_, name in zip(entry_vars[:i], entry_names[:i]) if name == entry_names[i]),
+                    )
+                    for i in range(len(entry_vars))
+                ),
             )
             if error is not None
         ),
@@ -204,6 +254,11 @@ def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str
     )
 
 
+def _entry_callback_name(entry: object) -> str | None:
+    callback_name: Final = entry.get("callback_name") if isinstance(entry, Mapping) else None
+    return callback_name if isinstance(callback_name, str) else None
+
+
 def _entry_callback_vars(entry: object) -> Mapping[str, str]:
     callback_vars: Final = entry.get("callback_vars") if isinstance(entry, Mapping) else None
     if not isinstance(callback_vars, Mapping):

litellm/proxy/management_endpoints/team_callback_endpoints.py

@@ -31,6 +31,7 @@ from litellm.proxy._types import (
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.callback_config_validation import (
     callback_config_error,
+    conflicting_internal_spans_error,
     conflicting_span_scope_error,
     cross_entry_family_error,
 )
@@ -285,6 +286,7 @@ async def add_team_callbacks(
         - langfuse_host: The host for the Langfuse callback
         - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)
         - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans
+        - otel_internal_spans: For langfuse_otel, arize, weave_otel and newrelic, "include" sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, "exclude" holds them back; defaults to litellm_settings.otel_tenant_internal_spans
         - gcs_bucket_name: The name of the GCS bucket
         - gcs_path_service_account: The path to the GCS service account
         - langsmith_api_key: The API key for the Langsmith callback
@@ -355,6 +357,16 @@ async def add_team_callbacks(
         scope_error: Final = conflicting_span_scope_error(data.callback_vars, stored_entry_vars)
         if scope_error is not None:
             raise _callback_config_error(scope_error)
+        internal_spans_error: Final = conflicting_internal_spans_error(
+            data.callback_vars,
+            tuple(
+                entry_vars
+                for entry, entry_vars in zip(stored_entries, stored_entry_vars)
+                if entry.get("callback_name") == data.callback_name
+            ),
+        )
+        if internal_spans_error is not None:
+            raise _callback_config_error(internal_spans_error)
         # One entry has to own a credential family end to end. The entries are
         # flattened into one dict before a request reads them, so an entry
         # naming only a destination would pair with a key written on another

litellm/types/utils.py

@@ -3612,6 +3612,8 @@ OPENAI_RESPONSE_HEADERS: Final = [
 
 OtelSpanScope = Literal["full", "llm_only"]
 OTEL_SPAN_SCOPES: Final[frozenset[str]] = frozenset(get_args(OtelSpanScope))
+OtelInternalSpans = Literal["include", "exclude"]
+OTEL_INTERNAL_SPAN_CHOICES: Final[frozenset[str]] = frozenset(get_args(OtelInternalSpans))
 
 
 class StandardCallbackDynamicParams(TypedDict, total=False):
@@ -3622,6 +3624,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False):
     langfuse_host: str | None
     langfuse_environment: ReadOnly[str | None]
     langfuse_span_scope: ReadOnly[OtelSpanScope | None]
+    otel_internal_spans: ReadOnly[OtelInternalSpans | None]
 
     # Langfuse prompt version
     langfuse_prompt_version: int | None

tests/test_litellm/integrations/otel/test_otel_v2_destinations.py

@@ -12,7 +12,7 @@ from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import SimpleSpanProcessor
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
-from opentelemetry.trace import Status, StatusCode
+from opentelemetry.trace import SpanKind, Status, StatusCode
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
@@ -1465,7 +1465,7 @@ TRACE_CONTROLS = MappingProxyType(
 
 def request_tree(provider: TracerProvider) -> None:
     tracer = get_tracer(provider, "litellm")
-    with tracer.start_as_current_span("POST /v1/chat/completions"):
+    with tracer.start_as_current_span("POST /v1/chat/completions", kind=SpanKind.SERVER):
         with tracer.start_as_current_span("auth /v1/chat/completions"):
             with tracer.start_as_current_span("postgres SELECT") as db:
                 db.set_attribute("db.system", "postgresql")
@@ -1901,6 +1901,191 @@ class TestSpanScope:
         assert saved.callback_vars["langfuse_span_scope"] == "llm_only"
 
 
+#: The proxy's own work inside the request: auth, the datastore calls, the spend write.
+INTERNAL_SPANS = frozenset({"auth /v1/chat/completions", "postgres SELECT", "redis GET", "cost_tracking"})
+TENANT_TREE = REQUEST_TREE - INTERNAL_SPANS
+EXCLUDING_DEST = LANGFUSE_DEST.model_copy(update={"internal_spans": "exclude"})
+LANGFUSE_TEAM_CREDS = MappingProxyType(
+    {"langfuse_public_key": "pk-team", "langfuse_secret_key": "sk-team", "langfuse_host": "http://team.local"}
+)
+COMPLETE_CREDENTIALS = MappingProxyType(
+    {
+        "langfuse_otel": LANGFUSE_TEAM_CREDS,
+        "arize": {"arize_space_id": "s", "arize_api_key": "k"},
+        "weave_otel": {"wandb_api_key": "k", "weave_project_id": "e/p"},
+        "newrelic": {"newrelic_api_key": "k"},
+    }
+)
+
+
+class TestInternalSpans:
+    @staticmethod
+    def _run(provider, destinations):
+        TestSpanScope._run(provider, destinations)
+
+    @staticmethod
+    def _fan_out(by_backend: Mapping[str, InMemorySpanExporter]) -> TracerProvider:
+        provider = TracerProvider()
+        provider.add_span_processor(
+            TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_backend[d.callback_name]))
+        )
+        return provider
+
+    @staticmethod
+    def _team_entry(**extra_vars: str) -> Mapping[str, object]:
+        return {
+            "callback_name": "langfuse_otel",
+            "callback_type": "success",
+            "callback_vars": {**LANGFUSE_TEAM_CREDS, **extra_vars},
+        }
+
+    def test_an_excluding_tenant_gets_the_request_root_and_its_own_calls_but_not_the_proxys_work(self, monkeypatch):
+        TestSpanScope._additive(monkeypatch)
+        operator, tenant = InMemorySpanExporter(), InMemorySpanExporter()
+
+        self._run(TestSpanScope._operator_provider(operator, tenant), (EXCLUDING_DEST,))
+
+        assert names(tenant) == TENANT_TREE
+        assert names(operator) == REQUEST_TREE, "the tenant's choice must not thin the operator's exporter"
+
+    def test_an_including_tenant_still_gets_the_whole_tree(self, monkeypatch):
+        TestSpanScope._additive(monkeypatch)
+        operator, tenant = InMemorySpanExporter(), InMemorySpanExporter()
+
+        self._run(TestSpanScope._operator_provider(operator, tenant), (LANGFUSE_DEST,))
+
+        assert names(tenant) == REQUEST_TREE
+
+    def test_the_kept_spans_stay_parented_to_the_request_root(self):
+        tenant = InMemorySpanExporter()
+
+        self._run(self._fan_out({"langfuse_otel": tenant}), (EXCLUDING_DEST,))
+
+        by_name = {span.name: span for span in tenant.get_finished_spans()}
+        root = by_name["POST /v1/chat/completions"]
+        assert root.parent is None
+        children = TENANT_TREE - {root.name}
+        assert {by_name[name].parent.span_id for name in children} == {root.context.span_id}
+
+    def test_two_destinations_of_one_request_decide_independently(self):
+        by_backend = {"langfuse_otel": InMemorySpanExporter(), "arize": InMemorySpanExporter()}
+        arize = OtelDestination(endpoint="https://otlp.arize.com", headers={"api_key": "k"}, callback_name="arize")
+
+        self._run(self._fan_out(by_backend), (EXCLUDING_DEST, arize))
+
+        assert names(by_backend["langfuse_otel"]) == TENANT_TREE
+        assert names(by_backend["arize"]) == REQUEST_TREE
+
+    def test_excluding_composes_with_llm_only(self):
+        tenant = InMemorySpanExporter()
+
+        excluding_llm_only = LLM_ONLY_DEST.model_copy(update={"internal_spans": "exclude"})
+
+        self._run(self._fan_out({"langfuse_otel": tenant}), (excluding_llm_only,))
+
+        assert names(tenant) == LLM_SPANS
+
+    def test_two_views_of_one_account_share_the_exporter(self):
+        built, tenant = [], InMemorySpanExporter()
+        provider = TracerProvider()
+
+        def factory(destination):
+            built.append(destination)
+            return SimpleSpanProcessor(tenant)
+
+        provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory))
+
+        self._run(provider, (EXCLUDING_DEST,))
+        assert names(tenant) == TENANT_TREE
+        tenant.clear()
+
+        self._run(provider, (LANGFUSE_DEST,))
+        assert names(tenant) == REQUEST_TREE
+        assert len(built) == 1, "the same account must not get a second exporter for a second internal_spans"
+
+    def test_a_team_callback_var_becomes_the_destinations_internal_spans(self, monkeypatch, allow_test_hosts):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        is_otel_v2_enabled.cache_clear()
+        auth = UserAPIKeyAuth(team_metadata={"logging": [self._team_entry(otel_internal_spans="exclude")]})
+
+        assert [d.internal_spans for d in resolve_tenant_otel_destinations(auth)] == ["exclude"]
+
+    def test_the_key_wins_over_the_team(self, monkeypatch, allow_test_hosts):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        is_otel_v2_enabled.cache_clear()
+        auth = UserAPIKeyAuth(
+            metadata={"logging": [self._team_entry(otel_internal_spans="include")]},
+            team_metadata={"logging": [self._team_entry(otel_internal_spans="exclude")]},
+        )
+
+        assert [d.internal_spans for d in resolve_tenant_otel_destinations(auth)] == ["include"]
+
+    def test_resolving_leaves_the_stored_callback_vars_alone(self, monkeypatch, allow_test_hosts):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        is_otel_v2_enabled.cache_clear()
+        entry = self._team_entry(otel_internal_spans="exclude")
+        before = dict(entry["callback_vars"])
+        auth = UserAPIKeyAuth(team_metadata={"logging": [entry]})
+
+        resolve_tenant_otel_destinations(auth)
+
+        assert entry["callback_vars"] == before
+
+    @pytest.mark.parametrize("callback_name", sorted(COMPLETE_CREDENTIALS))
+    def test_every_destination_backend_honours_the_var(self, callback_name, monkeypatch, allow_test_hosts):
+        assert callback_name in destination_capable_backends()
+        monkeypatch.setattr(litellm, "otel_tenant_internal_spans", None)
+        destination = destination_for(
+            callback_name, {**COMPLETE_CREDENTIALS[callback_name], "otel_internal_spans": "exclude"}
+        )
+
+        assert destination is not None and destination.internal_spans == "exclude"
+
+    def test_a_team_that_named_nothing_gets_the_system_default(self, monkeypatch, allow_test_hosts):
+        monkeypatch.setattr(litellm, "otel_tenant_internal_spans", "exclude")
+
+        assert destination_for("langfuse_otel", LANGFUSE_TEAM_CREDS).internal_spans == "exclude"
+
+    def test_the_env_var_sets_the_system_default(self, monkeypatch, allow_test_hosts):
+        monkeypatch.setattr(litellm, "otel_tenant_internal_spans", None)
+        monkeypatch.setenv("LITELLM_OTEL_TENANT_INTERNAL_SPANS", " Exclude\n")
+
+        assert destination_for("langfuse_otel", LANGFUSE_TEAM_CREDS).internal_spans == "exclude"
+
+    def test_a_team_asking_for_include_keeps_it_under_a_system_exclude(self, monkeypatch, allow_test_hosts):
+        monkeypatch.setattr(litellm, "otel_tenant_internal_spans", "exclude")
+
+        assert (
+            destination_for("langfuse_otel", {**LANGFUSE_TEAM_CREDS, "otel_internal_spans": "include"}).internal_spans
+            == "include"
+        )
+
+    @pytest.mark.parametrize("configured", [None, "", "everything"])
+    def test_anything_but_exclude_leaves_the_default_on_include(self, monkeypatch, allow_test_hosts, configured):
+        monkeypatch.setattr(litellm, "otel_tenant_internal_spans", configured)
+        monkeypatch.delenv("LITELLM_OTEL_TENANT_INTERNAL_SPANS", raising=False)
+
+        assert destination_for("langfuse_otel", LANGFUSE_TEAM_CREDS).internal_spans == "include"
+
+    @pytest.mark.parametrize("value", ["everything", "EXCLUDE", ""])
+    def test_an_unknown_value_is_rejected_when_the_callback_is_saved(self, value):
+        with pytest.raises(ValueError, match=r"Invalid otel_internal_spans .*must be one of \['exclude', 'include'\]"):
+            AddTeamCallback(
+                callback_name="langfuse_otel",
+                callback_type="success",
+                callback_vars={**LANGFUSE_TEAM_CREDS, "otel_internal_spans": value},
+            )
+
+    def test_a_known_value_is_accepted_when_the_callback_is_saved(self):
+        saved = AddTeamCallback(
+            callback_name="arize",
+            callback_type="success",
+            callback_vars={"arize_api_key": "k", "arize_space_id": "s", "otel_internal_spans": "exclude"},
+        )
+
+        assert saved.callback_vars["otel_internal_spans"] == "exclude"
+
+
 #: Anything that makes ``OpenTelemetryV2Config`` synthesize a real operator destination.
 _OTEL_SHORTHAND_ENV = (
     "OTEL_ENDPOINT",

tests/test_litellm/proxy/common_utils/test_callback_config_validation.py

@@ -2,6 +2,7 @@ import pytest
 
 from litellm.proxy.common_utils.callback_config_validation import (
     callback_config_error,
+    conflicting_internal_spans_error,
     conflicting_span_scope_error,
     cross_entry_family_error,
     logging_metadata_config_error,
@@ -101,6 +102,55 @@ def test_key_logging_entries_may_not_disagree_on_the_span_scope():
     assert logging_metadata_config_error(agreeing) is None
 
 
+@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"])
+def test_otel_internal_spans_is_accepted_on_every_destination_backend(callback_name):
+    assert callback_config_error(callback_name, {"otel_internal_spans": "exclude"}) is None
+    assert callback_config_error(callback_name, {"otel_internal_spans": "include"}) is None
+
+
+@pytest.mark.parametrize("bad", ["everything", "EXCLUDE", "no", ""])
+def test_callback_config_error_rejects_an_unknown_otel_internal_spans(bad):
+    error = callback_config_error("arize", {"otel_internal_spans": bad})
+    assert error is not None and "otel_internal_spans" in error and "exclude" in error
+
+
+@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None])
+def test_otel_internal_spans_on_a_callback_that_has_no_destination_is_rejected(callback_name):
+    error = callback_config_error(callback_name, {"otel_internal_spans": "exclude"})
+    assert error is not None and "otel_internal_spans" in error and "langfuse_otel" in error
+
+
+def test_key_logging_entries_of_one_backend_may_not_disagree_on_internal_spans():
+    def entry(callback_name, callback_type, value):
+        return {
+            "callback_name": callback_name,
+            "callback_type": callback_type,
+            "callback_vars": {"otel_internal_spans": value},
+        }
+
+    disagreeing = {"logging": [entry("arize", "success", "exclude"), entry("arize", "failure", "include")]}
+    error = logging_metadata_config_error(disagreeing)
+    assert error is not None and "otel_internal_spans" in error and "'exclude'" in error
+
+    two_backends = {"logging": [entry("arize", "success", "exclude"), entry("langfuse_otel", "success", "include")]}
+    assert logging_metadata_config_error(two_backends) is None
+
+
+@pytest.mark.parametrize(
+    "new_vars, stored, rejected",
+    [
+        ({"otel_internal_spans": "exclude"}, [{"otel_internal_spans": "include"}], True),
+        ({"otel_internal_spans": "exclude"}, [{"otel_internal_spans": "exclude"}], False),
+        ({"otel_internal_spans": "exclude"}, [{"arize_api_key": "k"}], False),
+        ({"arize_api_key": "k"}, [{"otel_internal_spans": "exclude"}], False),
+        (None, [{"otel_internal_spans": "exclude"}], False),
+    ],
+)
+def test_one_internal_spans_value_per_backend(new_vars, stored, rejected):
+    error = conflicting_internal_spans_error(new_vars, stored)
+    assert (error is not None) is rejected
+
+
 @pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"])
 @pytest.mark.parametrize("bad", ["1.5", "-0.1", "abc", "nan", "inf"])
 def test_callback_config_error_rejects_out_of_range_arize_sampling_rate(var, bad):

tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py

@@ -298,6 +298,19 @@ class TestLangfuseOtelCallbackConfig:
         assert frozenset(scope["options"]) == OTEL_SPAN_SCOPES
         assert scope["required"] is False
 
+    @pytest.mark.parametrize("callback_id", ["langfuse_otel", "arize", "newrelic"])
+    def test_internal_spans_is_a_select_over_exactly_the_choices_the_validator_accepts(self, callback_id):
+        from litellm.types.utils import OTEL_INTERNAL_SPAN_CHOICES
+
+        client = TestClient(app)
+        response = client.get("/callbacks/configs", headers={"Authorization": "Bearer sk-1234"})
+        assert response.status_code == 200
+        config = next(config for config in response.json() if config.get("id") == callback_id)
+        internal_spans = config["dynamic_params"]["otel_internal_spans"]
+        assert internal_spans["type"] == "select"
+        assert frozenset(internal_spans["options"]) == OTEL_INTERNAL_SPAN_CHOICES
+        assert internal_spans["required"] is False
+
 
 class TestNewRelicTeamCallbackValidation:
     def _data(self, callback_vars):

tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py

@@ -1674,6 +1674,62 @@ async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller
     patched_prisma.db.litellm_teamtable.update.assert_awaited_once()
 
 
+@pytest.mark.asyncio
+async def test_a_second_entry_may_not_flip_internal_spans_but_another_backend_may(patched_prisma):
+    """The var merges per backend, so a second langfuse_otel entry saying include next to
+    a stored exclude would export whichever is stored last and is refused, while an arize
+    entry saying include is a different destination and is stored as written."""
+    patched_prisma.get_data = AsyncMock(
+        return_value=_team_row(
+            metadata={
+                "logging": [
+                    {
+                        "callback_name": "langfuse_otel",
+                        "callback_type": "success",
+                        "callback_vars": {
+                            "langfuse_public_key": "pk",
+                            "langfuse_secret_key": "sk",
+                            "otel_internal_spans": "exclude",
+                        },
+                    }
+                ]
+            }
+        )
+    )
+    with pytest.raises(HTTPException) as exc:
+        await add_team_callbacks(
+            data=AddTeamCallback(
+                callback_name="langfuse_otel",
+                callback_type="failure",
+                callback_vars={
+                    "langfuse_public_key": "pk",
+                    "langfuse_secret_key": "sk",
+                    "otel_internal_spans": "include",
+                },
+            ),
+            http_request=Mock(spec=Request),
+            team_id="team-victim",
+            user_api_key_dict=_admin_auth(),
+        )
+    assert exc.value.status_code == 400
+    assert "otel_internal_spans" in str(exc.value.detail) and "'exclude'" in str(exc.value.detail)
+    patched_prisma.db.litellm_teamtable.update.assert_not_called()
+
+    await add_team_callbacks(
+        data=AddTeamCallback(
+            callback_name="arize",
+            callback_type="success",
+            callback_vars={"arize_api_key": "k", "arize_space_id": "s", "otel_internal_spans": "include"},
+        ),
+        http_request=Mock(spec=Request),
+        team_id="team-victim",
+        user_api_key_dict=_admin_auth(),
+    )
+    patched_prisma.db.litellm_teamtable.update.assert_awaited_once()
+    saved = json.loads(patched_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"])
+    assert [entry["callback_vars"]["otel_internal_spans"] for entry in saved["logging"]] == ["exclude", "include"]
+
+
 @pytest.mark.asyncio
 async def test_add_team_callbacks_rejects_out_of_range_arize_sampling_rate(patched_prisma):
     data = AddTeamCallback(

ui/litellm-dashboard/src/components/callback_info_helpers.tsx

@@ -32,6 +32,10 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
       arize_space_id: "password",
       arize_success_sampling_rate: "number",
       arize_error_sampling_rate: "number",
+      otel_internal_spans: "select",
+    },
+    dynamic_param_options: {
+      otel_internal_spans: ["include", "exclude"],
     },
     description: "Arize Logging Integration",
   },
@@ -90,6 +94,10 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
     dynamic_params: {
       newrelic_api_key: "password",
       newrelic_region: "text",
+      otel_internal_spans: "select",
+    },
+    dynamic_param_options: {
+      otel_internal_spans: ["include", "exclude"],
     },
     description: "New Relic Logging Integration",
   },
@@ -128,9 +136,11 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
       langfuse_host: "text",
       langfuse_environment: "text",
       langfuse_span_scope: "select",
+      otel_internal_spans: "select",
     },
     dynamic_param_options: {
       langfuse_span_scope: ["full", "llm_only"],
+      otel_internal_spans: ["include", "exclude"],
     },
     description: "Langfuse v3 OTEL Logging Integration",
   },

ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx

@@ -239,6 +239,29 @@ describe("LoggingSettings", () => {
     ]);
   });
 
+  it("offers the OTEL destination internal spans as a pick between include and exclude", async () => {
+    const user = userEvent.setup({ delay: null });
+    const mockOnChange = vi.fn();
+    const initialValue = [
+      {
+        callback_name: "arize",
+        callback_type: "success",
+        callback_vars: {},
+      },
+    ];
+
+    renderWithProviders(<LoggingSettings value={initialValue} onChange={mockOnChange} />);
+
+    expect(screen.queryByPlaceholderText("os.environ/OTEL_INTERNAL_SPANS")).not.toBeInTheDocument();
+    await user.click(screen.getByRole("combobox", { name: "otel internal spans" }));
+    expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual(["include", "exclude"]);
+    await user.click(screen.getByRole("option", { name: "exclude" }));
+
+    expect(mockOnChange).toHaveBeenCalledWith([
+      expect.objectContaining({ callback_vars: expect.objectContaining({ otel_internal_spans: "exclude" }) }),
+    ]);
+  });
+
   it("renders sampling rate inputs for the Arize callback and records changes", () => {
     const mockOnChange = vi.fn();
 

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -16322,6 +16322,7 @@ export interface paths {
          *         - langfuse_host: The host for the Langfuse callback
          *         - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)
          *         - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans
+         *         - otel_internal_spans: For langfuse_otel, arize, weave_otel and newrelic, "include" sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, "exclude" holds them back; defaults to litellm_settings.otel_tenant_internal_spans
          *         - gcs_bucket_name: The name of the GCS bucket
          *         - gcs_path_service_account: The path to the GCS service account
          *         - langsmith_api_key: The API key for the Langsmith callback