From 4a92e3055fed3f05521d0fc8b8be3fc18f7cdd04 Mon Sep 17 00:00:00 2001 From: mrinal Date: Tue, 22 Sep 2026 22:46:34 +0000 Subject: [PATCH] feat(otel v2): per-destination otel_internal_spans to hold back the proxy's SERVICE and DB_CALL spans from tenant exporters A key or team that points langfuse_otel, arize, weave_otel or newrelic at its own account can now set otel_internal_spans to exclude, and the TenantFanOutSpanProcessor stops forwarding the proxy's own auth, Redis, Postgres and spend-write spans to that destination. The request root and the tenant's model, guardrail and MCP spans still go, and the operator's exporter still receives the full tree. The default stays include, with litellm_settings.otel_tenant_internal_spans (or LITELLM_OTEL_TENANT_INTERNAL_SPANS) as the system-wide default for destinations that name no value. The team and key Logging Settings forms render the field as a select; the global Add Callback form does not, since the operator exporter cannot honour it. Resolves LIT-8245 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 3 + litellm/integrations/callback_configs.json | 21 ++ .../integrations/otel/model/destination.py | 15 +- litellm/integrations/otel/plumbing/context.py | 19 ++ .../integrations/otel/plumbing/providers.py | 20 +- .../integrations/otel/presets/destinations.py | 9 +- .../initialize_dynamic_callback_params.py | 12 +- litellm/proxy/_types.py | 3 + .../callback_config_validation.py | 63 +++++- .../team_callback_endpoints.py | 12 ++ litellm/types/utils.py | 3 + .../otel/test_otel_v2_destinations.py | 189 +++++++++++++++++- .../test_callback_config_validation.py | 50 +++++ .../test_callback_management_endpoints.py | 13 ++ .../test_team_callback_endpoints.py | 56 ++++++ .../src/components/callback_info_helpers.tsx | 10 + .../components/team/LoggingSettings.test.tsx | 23 +++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 1 + 18 files changed, 507 insertions(+), 15 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 471b273f00d..748d02d453c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -328,6 +328,9 @@ provider_url_destination_allowed_hosts: List[str] = [] #: "override" (default) or "additive": whether a key or team destination replaces #: the operator's exporter for that backend or exports alongside it. otel_tenant_destination_mode: str | None = None +#: "include" (default) or "exclude": whether a key or team destination receives the +#: proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres, spend writes). +otel_tenant_internal_spans: str | None = None ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance disable_streaming_logging: bool = False disable_token_counter: bool = False diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 5bd8aca55fa..b227f27b47a 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -16,6 +16,13 @@ "ui_name": "Space ID", "description": "Arize Space ID to identify your workspace", "required": true + }, + "otel_internal_spans": { + "type": "select", + "ui_name": "Internal Spans", + "description": "include sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, exclude holds them back", + "options": ["include", "exclude"], + "required": false } }, "description": "Arize Logging Integration" @@ -266,6 +273,13 @@ "description": "full sends the whole request trace, llm_only sends just the model-call spans", "options": ["full", "llm_only"], "required": false + }, + "otel_internal_spans": { + "type": "select", + "ui_name": "Internal Spans", + "description": "include sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, exclude holds them back", + "options": ["include", "exclude"], + "required": false } }, "description": "Langfuse v3 OTEL Logging Integration" @@ -326,6 +340,13 @@ "ui_name": "New Relic Region (us or eu)", "description": "Data center region for this team's account. Defaults to us.", "required": false + }, + "otel_internal_spans": { + "type": "select", + "ui_name": "Internal Spans", + "description": "include sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, exclude holds them back", + "options": ["include", "exclude"], + "required": false } }, "description": "New Relic AI Monitoring Integration" diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index c9c035f24a1..096b49159dc 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -10,7 +10,7 @@ from urllib.parse import quote from pydantic import BaseModel, ConfigDict, Field -from litellm.types.utils import OtelSpanScope +from litellm.types.utils import OtelInternalSpans, OtelSpanScope class OtelDestination(BaseModel): @@ -31,6 +31,13 @@ class OtelDestination(BaseModel): default="full", description="``llm_only`` keeps just the model-call spans; the rest of the request tree is not forwarded.", ) + internal_spans: OtelInternalSpans = Field( + default="include", + description=( + "``exclude`` holds back the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres, " + "spend writes); the request root and the tenant's own model, guardrail and MCP spans still go." + ), + ) def header_string(self) -> str: """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects. @@ -45,9 +52,9 @@ class OtelDestination(BaseModel): def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: """Identity for processor reuse, so one destination means one exporter. - ``span_scope`` is left out on purpose: the scope decides which spans reach the - processor, not how the processor exports them, so a full and an ``llm_only`` - view of the same account share one exporter. + ``span_scope`` and ``internal_spans`` are left out on purpose: they decide which + spans reach the processor, not how the processor exports them, so a full and an + ``llm_only`` view of the same account share one exporter. """ return ( self.endpoint, diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index 19243d64c64..506d79598eb 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -21,6 +21,7 @@ from opentelemetry.trace.propagation.tracecontext import ( ) from litellm.integrations.otel.model.semconv import HTTP +from litellm.types.utils import OtelInternalSpans if TYPE_CHECKING: from litellm.integrations.otel.model.destination import OtelDestination @@ -400,6 +401,24 @@ def tenant_destinations_are_additive() -> bool: return isinstance(configured, str) and configured.strip().lower() == ADDITIVE_DESTINATION_MODE +EXCLUDE_INTERNAL_SPANS: Final = "exclude" +OTEL_TENANT_INTERNAL_SPANS_ENV: Final = "LITELLM_OTEL_TENANT_INTERNAL_SPANS" + + +def tenant_internal_spans_default() -> OtelInternalSpans: + """The ``internal_spans`` a tenant destination gets when its own callback vars name none. + + Include is the default, so a destination keeps the whole request tree unless the + operator or the team asks for the proxy's own SERVICE and DB_CALL spans to stay home. + """ + import litellm + + configured: Final = litellm.otel_tenant_internal_spans or os.environ.get(OTEL_TENANT_INTERNAL_SPANS_ENV) + if isinstance(configured, str) and configured.strip().lower() == EXCLUDE_INTERNAL_SPANS: + return "exclude" + return "include" + + def destination_backends() -> frozenset[str]: """Backends this request resolved a tenant destination for. diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index d52736a1303..2752bd473ec 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -440,6 +440,22 @@ def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool: return scope == "full" or is_llm_call_span(span) +def _is_internal_span(span: ReadableSpan) -> bool: + """Whether ``span`` is the proxy's own work rather than the request or the tenant's call. + + The request root is the SERVER span; the model call, the MCP call and the guardrail + each carry one of the tenant-owned keys. Everything else the proxy emits inside a + request is a SERVICE or DB_CALL span: auth, Redis, Postgres, spend writes. + """ + return span.kind is not SpanKind.SERVER and not _is_tenant_owned_span(span.attributes or _NO_ATTRIBUTES) + + +def _forwarded(span: ReadableSpan, destination: "OtelDestination") -> bool: + if not _in_scope(span, destination.span_scope): + return False + return destination.internal_spans == "include" or not _is_internal_span(span) + + def _scoped(span: ReadableSpan, scope: "OtelSpanScope") -> ReadableSpan: """Under ``llm_only`` the model call is the only span the exporter gets, so it goes out as the trace's root (its parent is the request span that is held back) and, unless the caller named the @@ -570,9 +586,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): def on_end(self, span: ReadableSpan) -> None: suppressed: Final = suppressed_backends() for destination in request_destinations(): - if self._operator_already_writes(span, destination, suppressed) or not _in_scope( - span, destination.span_scope - ): + if self._operator_already_writes(span, destination, suppressed) or not _forwarded(span, destination): continue processor = self._acquire(destination) # rebind-ok: loop variable; pyright forbids Final in a loop if processor is None: diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 63801e623af..bde1ecacc5e 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -14,8 +14,9 @@ from typing import Final import litellm from litellm._logging import verbose_logger from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.plumbing.context import tenant_internal_spans_default from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host -from litellm.types.utils import OtelSpanScope, StandardCallbackDynamicParams +from litellm.types.utils import OtelInternalSpans, OtelSpanScope, StandardCallbackDynamicParams #: An endpoint plus the OTLP transport to reach it with, or ``None`` when the backend #: names no destination. The transport is ``None`` where the backend has only one. @@ -117,6 +118,11 @@ def _span_scope(callback_name: str, params: StandardCallbackDynamicParams) -> Ot return params.get("langfuse_span_scope") or "full" +def _internal_spans(params: StandardCallbackDynamicParams) -> OtelInternalSpans: + configured: Final = params.get("otel_internal_spans") + return tenant_internal_spans_default() if configured is None else configured + + def destination_capable_backends() -> frozenset[str]: """Backends a key or team can point at its own account.""" from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK @@ -156,4 +162,5 @@ def destination_for( callback_name=callback_name, protocol=protocol, span_scope=_span_scope(callback_name, params), + internal_spans=_internal_spans(params), ) diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 00ab05aba77..c2cd813e021 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -4,7 +4,12 @@ from contextlib import contextmanager from contextvars import ContextVar from typing import Any, Final -from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams +from litellm.types.utils import ( + OTEL_INTERNAL_SPAN_CHOICES, + OTEL_SPAN_SCOPES, + TRUSTED_CALLBACK_VARS_FIELD, + StandardCallbackDynamicParams, +) _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata") _inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar( @@ -81,6 +86,11 @@ def validate_langfuse_span_scope_value(value: str) -> None: raise ValueError(f"Invalid langfuse_span_scope {value!r}: must be one of {sorted(OTEL_SPAN_SCOPES)}") +def validate_otel_internal_spans_value(value: str) -> None: + if value not in OTEL_INTERNAL_SPAN_CHOICES: + raise ValueError(f"Invalid otel_internal_spans {value!r}: must be one of {sorted(OTEL_INTERNAL_SPAN_CHOICES)}") + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 76df8790b92..ff160b888cf 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_langfuse_environment_value, validate_langfuse_span_scope_value, validate_no_callback_env_reference, + validate_otel_internal_spans_value, ) from litellm.types.agents import AgentCaller from litellm.types.integrations.compression_interception import ( @@ -2258,6 +2259,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): validate_langfuse_environment_value(callback_vars[key]) if key == "langfuse_span_scope": validate_langfuse_span_scope_value(callback_vars[key]) + if key == "otel_internal_spans": + validate_otel_internal_spans_value(callback_vars[key]) return values diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 6a0fcbe0bb3..d2071ef83e3 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -14,6 +14,7 @@ _NEWRELIC_CALLBACK: Final = "newrelic" _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" _LANGFUSE_SPAN_SCOPE_VAR: Final = "langfuse_span_scope" +_OTEL_INTERNAL_SPANS_VAR: Final = "otel_internal_spans" _ARIZE_CALLBACK: Final = "arize" _ARIZE_SAMPLING_RATE_VARS: Final[frozenset[str]] = frozenset( {"arize_success_sampling_rate", "arize_error_sampling_rate"} @@ -31,6 +32,9 @@ def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, ) if langfuse_error is not None: return langfuse_error + internal_spans_error: Final = _otel_internal_spans_error(callback_name, callback_vars) + if internal_spans_error is not None: + return internal_spans_error if callback_name != _NEWRELIC_CALLBACK: return None return _newrelic_config_error(callback_vars) @@ -75,6 +79,27 @@ def _langfuse_span_scope_error(callback_name: str | None, callback_vars: Mapping return None +def _otel_internal_spans_error(callback_name: str | None, callback_vars: Mapping[str, str]) -> str | None: + value: Final = callback_vars.get(_OTEL_INTERNAL_SPANS_VAR) + if value is None: + return None + from litellm.integrations.otel.presets.destinations import destination_capable_backends + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_otel_internal_spans_value, + ) + + if callback_name not in destination_capable_backends(): + return ( + f"{_OTEL_INTERNAL_SPANS_VAR} applies to the OTEL destination callbacks only " + f"({', '.join(sorted(destination_capable_backends()))}), not {callback_name!r}" + ) + try: + validate_otel_internal_spans_value(value) + except ValueError as e: + return str(e) + return None + + # Which credential family a dynamic variable belongs to. The families are the # integrations that share one account: every langfuse_* variable configures the # same Langfuse project whether it rides the classic callback or the OTel one, @@ -169,15 +194,32 @@ def conflicting_span_scope_error( callback_vars: Mapping[str, str] | None, stored_vars_by_entry: Sequence[Mapping[str, str]], ) -> str | None: - incoming: Final = None if callback_vars is None else callback_vars.get(_LANGFUSE_SPAN_SCOPE_VAR) + return _conflicting_var_error(_LANGFUSE_SPAN_SCOPE_VAR, callback_vars, stored_vars_by_entry) + + +def conflicting_internal_spans_error( + callback_vars: Mapping[str, str] | None, + stored_vars_by_entry: Sequence[Mapping[str, str]], +) -> str | None: + """``stored_vars_by_entry`` must hold only the entries of the same callback: the + request merges the var per backend, so two backends may legitimately disagree.""" + return _conflicting_var_error(_OTEL_INTERNAL_SPANS_VAR, callback_vars, stored_vars_by_entry) + + +def _conflicting_var_error( + var: str, + callback_vars: Mapping[str, str] | None, + stored_vars_by_entry: Sequence[Mapping[str, str]], +) -> str | None: + incoming: Final = None if callback_vars is None else callback_vars.get(var) if incoming is None: return None return next( ( - f"{_LANGFUSE_SPAN_SCOPE_VAR} is already set to {stored!r} by another callback entry. " - f"Every entry shares one scope: remove that entry or send the same value." + f"{var} is already set to {stored!r} by another callback entry. " + f"Every entry shares one value: remove that entry or send the same value." for entry in stored_vars_by_entry - if (stored := entry.get(_LANGFUSE_SPAN_SCOPE_VAR)) not in (None, incoming) + if (stored := entry.get(var)) not in (None, incoming) ), None, ) @@ -191,12 +233,20 @@ def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): return None entry_vars: Final = tuple(_entry_callback_vars(entry) for entry in entries) + entry_names: Final = tuple(_entry_callback_name(entry) for entry in entries) return next( ( error for error in ( *(_logging_entry_error(entry) for entry in entries), *(conflicting_span_scope_error(entry_vars[i], entry_vars[:i]) for i in range(len(entry_vars))), + *( + conflicting_internal_spans_error( + entry_vars[i], + tuple(vars_ for vars_, name in zip(entry_vars[:i], entry_names[:i]) if name == entry_names[i]), + ) + for i in range(len(entry_vars)) + ), ) if error is not None ), @@ -204,6 +254,11 @@ def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str ) +def _entry_callback_name(entry: object) -> str | None: + callback_name: Final = entry.get("callback_name") if isinstance(entry, Mapping) else None + return callback_name if isinstance(callback_name, str) else None + + def _entry_callback_vars(entry: object) -> Mapping[str, str]: callback_vars: Final = entry.get("callback_vars") if isinstance(entry, Mapping) else None if not isinstance(callback_vars, Mapping): diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 4091d69e44e..c678c51083d 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -31,6 +31,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_internal_spans_error, conflicting_span_scope_error, cross_entry_family_error, ) @@ -285,6 +286,7 @@ async def add_team_callbacks( - langfuse_host: The host for the Langfuse callback - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + - otel_internal_spans: For langfuse_otel, arize, weave_otel and newrelic, "include" sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, "exclude" holds them back; defaults to litellm_settings.otel_tenant_internal_spans - gcs_bucket_name: The name of the GCS bucket - gcs_path_service_account: The path to the GCS service account - langsmith_api_key: The API key for the Langsmith callback @@ -355,6 +357,16 @@ async def add_team_callbacks( scope_error: Final = conflicting_span_scope_error(data.callback_vars, stored_entry_vars) if scope_error is not None: raise _callback_config_error(scope_error) + internal_spans_error: Final = conflicting_internal_spans_error( + data.callback_vars, + tuple( + entry_vars + for entry, entry_vars in zip(stored_entries, stored_entry_vars) + if entry.get("callback_name") == data.callback_name + ), + ) + if internal_spans_error is not None: + raise _callback_config_error(internal_spans_error) # One entry has to own a credential family end to end. The entries are # flattened into one dict before a request reads them, so an entry # naming only a destination would pair with a key written on another diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f2c8f0e7044..d1a10e0d514 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3612,6 +3612,8 @@ OPENAI_RESPONSE_HEADERS: Final = [ OtelSpanScope = Literal["full", "llm_only"] OTEL_SPAN_SCOPES: Final[frozenset[str]] = frozenset(get_args(OtelSpanScope)) +OtelInternalSpans = Literal["include", "exclude"] +OTEL_INTERNAL_SPAN_CHOICES: Final[frozenset[str]] = frozenset(get_args(OtelInternalSpans)) class StandardCallbackDynamicParams(TypedDict, total=False): @@ -3622,6 +3624,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False): langfuse_host: str | None langfuse_environment: ReadOnly[str | None] langfuse_span_scope: ReadOnly[OtelSpanScope | None] + otel_internal_spans: ReadOnly[OtelInternalSpans | None] # Langfuse prompt version langfuse_prompt_version: int | None diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 9cb3dbb9deb..f94ee8ce3cf 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -12,7 +12,7 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from opentelemetry.trace import Status, StatusCode +from opentelemetry.trace import SpanKind, Status, StatusCode import litellm from litellm.integrations.custom_logger import CustomLogger @@ -1465,7 +1465,7 @@ TRACE_CONTROLS = MappingProxyType( def request_tree(provider: TracerProvider) -> None: tracer = get_tracer(provider, "litellm") - with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("POST /v1/chat/completions", kind=SpanKind.SERVER): with tracer.start_as_current_span("auth /v1/chat/completions"): with tracer.start_as_current_span("postgres SELECT") as db: db.set_attribute("db.system", "postgresql") @@ -1901,6 +1901,191 @@ class TestSpanScope: assert saved.callback_vars["langfuse_span_scope"] == "llm_only" +#: The proxy's own work inside the request: auth, the datastore calls, the spend write. +INTERNAL_SPANS = frozenset({"auth /v1/chat/completions", "postgres SELECT", "redis GET", "cost_tracking"}) +TENANT_TREE = REQUEST_TREE - INTERNAL_SPANS +EXCLUDING_DEST = LANGFUSE_DEST.model_copy(update={"internal_spans": "exclude"}) +LANGFUSE_TEAM_CREDS = MappingProxyType( + {"langfuse_public_key": "pk-team", "langfuse_secret_key": "sk-team", "langfuse_host": "http://team.local"} +) +COMPLETE_CREDENTIALS = MappingProxyType( + { + "langfuse_otel": LANGFUSE_TEAM_CREDS, + "arize": {"arize_space_id": "s", "arize_api_key": "k"}, + "weave_otel": {"wandb_api_key": "k", "weave_project_id": "e/p"}, + "newrelic": {"newrelic_api_key": "k"}, + } +) + + +class TestInternalSpans: + @staticmethod + def _run(provider, destinations): + TestSpanScope._run(provider, destinations) + + @staticmethod + def _fan_out(by_backend: Mapping[str, InMemorySpanExporter]) -> TracerProvider: + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_backend[d.callback_name])) + ) + return provider + + @staticmethod + def _team_entry(**extra_vars: str) -> Mapping[str, object]: + return { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {**LANGFUSE_TEAM_CREDS, **extra_vars}, + } + + def test_an_excluding_tenant_gets_the_request_root_and_its_own_calls_but_not_the_proxys_work(self, monkeypatch): + TestSpanScope._additive(monkeypatch) + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(TestSpanScope._operator_provider(operator, tenant), (EXCLUDING_DEST,)) + + assert names(tenant) == TENANT_TREE + assert names(operator) == REQUEST_TREE, "the tenant's choice must not thin the operator's exporter" + + def test_an_including_tenant_still_gets_the_whole_tree(self, monkeypatch): + TestSpanScope._additive(monkeypatch) + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(TestSpanScope._operator_provider(operator, tenant), (LANGFUSE_DEST,)) + + assert names(tenant) == REQUEST_TREE + + def test_the_kept_spans_stay_parented_to_the_request_root(self): + tenant = InMemorySpanExporter() + + self._run(self._fan_out({"langfuse_otel": tenant}), (EXCLUDING_DEST,)) + + by_name = {span.name: span for span in tenant.get_finished_spans()} + root = by_name["POST /v1/chat/completions"] + assert root.parent is None + children = TENANT_TREE - {root.name} + assert {by_name[name].parent.span_id for name in children} == {root.context.span_id} + + def test_two_destinations_of_one_request_decide_independently(self): + by_backend = {"langfuse_otel": InMemorySpanExporter(), "arize": InMemorySpanExporter()} + arize = OtelDestination(endpoint="https://otlp.arize.com", headers={"api_key": "k"}, callback_name="arize") + + self._run(self._fan_out(by_backend), (EXCLUDING_DEST, arize)) + + assert names(by_backend["langfuse_otel"]) == TENANT_TREE + assert names(by_backend["arize"]) == REQUEST_TREE + + def test_excluding_composes_with_llm_only(self): + tenant = InMemorySpanExporter() + + excluding_llm_only = LLM_ONLY_DEST.model_copy(update={"internal_spans": "exclude"}) + + self._run(self._fan_out({"langfuse_otel": tenant}), (excluding_llm_only,)) + + assert names(tenant) == LLM_SPANS + + def test_two_views_of_one_account_share_the_exporter(self): + built, tenant = [], InMemorySpanExporter() + provider = TracerProvider() + + def factory(destination): + built.append(destination) + return SimpleSpanProcessor(tenant) + + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + + self._run(provider, (EXCLUDING_DEST,)) + assert names(tenant) == TENANT_TREE + tenant.clear() + + self._run(provider, (LANGFUSE_DEST,)) + assert names(tenant) == REQUEST_TREE + assert len(built) == 1, "the same account must not get a second exporter for a second internal_spans" + + def test_a_team_callback_var_becomes_the_destinations_internal_spans(self, monkeypatch, allow_test_hosts): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth(team_metadata={"logging": [self._team_entry(otel_internal_spans="exclude")]}) + + assert [d.internal_spans for d in resolve_tenant_otel_destinations(auth)] == ["exclude"] + + def test_the_key_wins_over_the_team(self, monkeypatch, allow_test_hosts): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + metadata={"logging": [self._team_entry(otel_internal_spans="include")]}, + team_metadata={"logging": [self._team_entry(otel_internal_spans="exclude")]}, + ) + + assert [d.internal_spans for d in resolve_tenant_otel_destinations(auth)] == ["include"] + + def test_resolving_leaves_the_stored_callback_vars_alone(self, monkeypatch, allow_test_hosts): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + entry = self._team_entry(otel_internal_spans="exclude") + before = dict(entry["callback_vars"]) + auth = UserAPIKeyAuth(team_metadata={"logging": [entry]}) + + resolve_tenant_otel_destinations(auth) + + assert entry["callback_vars"] == before + + @pytest.mark.parametrize("callback_name", sorted(COMPLETE_CREDENTIALS)) + def test_every_destination_backend_honours_the_var(self, callback_name, monkeypatch, allow_test_hosts): + assert callback_name in destination_capable_backends() + monkeypatch.setattr(litellm, "otel_tenant_internal_spans", None) + destination = destination_for( + callback_name, {**COMPLETE_CREDENTIALS[callback_name], "otel_internal_spans": "exclude"} + ) + + assert destination is not None and destination.internal_spans == "exclude" + + def test_a_team_that_named_nothing_gets_the_system_default(self, monkeypatch, allow_test_hosts): + monkeypatch.setattr(litellm, "otel_tenant_internal_spans", "exclude") + + assert destination_for("langfuse_otel", LANGFUSE_TEAM_CREDS).internal_spans == "exclude" + + def test_the_env_var_sets_the_system_default(self, monkeypatch, allow_test_hosts): + monkeypatch.setattr(litellm, "otel_tenant_internal_spans", None) + monkeypatch.setenv("LITELLM_OTEL_TENANT_INTERNAL_SPANS", " Exclude\n") + + assert destination_for("langfuse_otel", LANGFUSE_TEAM_CREDS).internal_spans == "exclude" + + def test_a_team_asking_for_include_keeps_it_under_a_system_exclude(self, monkeypatch, allow_test_hosts): + monkeypatch.setattr(litellm, "otel_tenant_internal_spans", "exclude") + + assert ( + destination_for("langfuse_otel", {**LANGFUSE_TEAM_CREDS, "otel_internal_spans": "include"}).internal_spans + == "include" + ) + + @pytest.mark.parametrize("configured", [None, "", "everything"]) + def test_anything_but_exclude_leaves_the_default_on_include(self, monkeypatch, allow_test_hosts, configured): + monkeypatch.setattr(litellm, "otel_tenant_internal_spans", configured) + monkeypatch.delenv("LITELLM_OTEL_TENANT_INTERNAL_SPANS", raising=False) + + assert destination_for("langfuse_otel", LANGFUSE_TEAM_CREDS).internal_spans == "include" + + @pytest.mark.parametrize("value", ["everything", "EXCLUDE", ""]) + def test_an_unknown_value_is_rejected_when_the_callback_is_saved(self, value): + with pytest.raises(ValueError, match=r"Invalid otel_internal_spans .*must be one of \['exclude', 'include'\]"): + AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={**LANGFUSE_TEAM_CREDS, "otel_internal_spans": value}, + ) + + def test_a_known_value_is_accepted_when_the_callback_is_saved(self): + saved = AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_api_key": "k", "arize_space_id": "s", "otel_internal_spans": "exclude"}, + ) + + assert saved.callback_vars["otel_internal_spans"] == "exclude" + + #: Anything that makes ``OpenTelemetryV2Config`` synthesize a real operator destination. _OTEL_SHORTHAND_ENV = ( "OTEL_ENDPOINT", diff --git a/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py b/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py index 33ae986a511..f76152861d1 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py @@ -2,6 +2,7 @@ import pytest from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_internal_spans_error, conflicting_span_scope_error, cross_entry_family_error, logging_metadata_config_error, @@ -101,6 +102,55 @@ def test_key_logging_entries_may_not_disagree_on_the_span_scope(): assert logging_metadata_config_error(agreeing) is None +@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"]) +def test_otel_internal_spans_is_accepted_on_every_destination_backend(callback_name): + assert callback_config_error(callback_name, {"otel_internal_spans": "exclude"}) is None + assert callback_config_error(callback_name, {"otel_internal_spans": "include"}) is None + + +@pytest.mark.parametrize("bad", ["everything", "EXCLUDE", "no", ""]) +def test_callback_config_error_rejects_an_unknown_otel_internal_spans(bad): + error = callback_config_error("arize", {"otel_internal_spans": bad}) + assert error is not None and "otel_internal_spans" in error and "exclude" in error + + +@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None]) +def test_otel_internal_spans_on_a_callback_that_has_no_destination_is_rejected(callback_name): + error = callback_config_error(callback_name, {"otel_internal_spans": "exclude"}) + assert error is not None and "otel_internal_spans" in error and "langfuse_otel" in error + + +def test_key_logging_entries_of_one_backend_may_not_disagree_on_internal_spans(): + def entry(callback_name, callback_type, value): + return { + "callback_name": callback_name, + "callback_type": callback_type, + "callback_vars": {"otel_internal_spans": value}, + } + + disagreeing = {"logging": [entry("arize", "success", "exclude"), entry("arize", "failure", "include")]} + error = logging_metadata_config_error(disagreeing) + assert error is not None and "otel_internal_spans" in error and "'exclude'" in error + + two_backends = {"logging": [entry("arize", "success", "exclude"), entry("langfuse_otel", "success", "include")]} + assert logging_metadata_config_error(two_backends) is None + + +@pytest.mark.parametrize( + "new_vars, stored, rejected", + [ + ({"otel_internal_spans": "exclude"}, [{"otel_internal_spans": "include"}], True), + ({"otel_internal_spans": "exclude"}, [{"otel_internal_spans": "exclude"}], False), + ({"otel_internal_spans": "exclude"}, [{"arize_api_key": "k"}], False), + ({"arize_api_key": "k"}, [{"otel_internal_spans": "exclude"}], False), + (None, [{"otel_internal_spans": "exclude"}], False), + ], +) +def test_one_internal_spans_value_per_backend(new_vars, stored, rejected): + error = conflicting_internal_spans_error(new_vars, stored) + assert (error is not None) is rejected + + @pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"]) @pytest.mark.parametrize("bad", ["1.5", "-0.1", "abc", "nan", "inf"]) def test_callback_config_error_rejects_out_of_range_arize_sampling_rate(var, bad): diff --git a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py index 5c5cfd0814d..6fd9b6f9b34 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py @@ -298,6 +298,19 @@ class TestLangfuseOtelCallbackConfig: assert frozenset(scope["options"]) == OTEL_SPAN_SCOPES assert scope["required"] is False + @pytest.mark.parametrize("callback_id", ["langfuse_otel", "arize", "newrelic"]) + def test_internal_spans_is_a_select_over_exactly_the_choices_the_validator_accepts(self, callback_id): + from litellm.types.utils import OTEL_INTERNAL_SPAN_CHOICES + + client = TestClient(app) + response = client.get("/callbacks/configs", headers={"Authorization": "Bearer sk-1234"}) + assert response.status_code == 200 + config = next(config for config in response.json() if config.get("id") == callback_id) + internal_spans = config["dynamic_params"]["otel_internal_spans"] + assert internal_spans["type"] == "select" + assert frozenset(internal_spans["options"]) == OTEL_INTERNAL_SPAN_CHOICES + assert internal_spans["required"] is False + class TestNewRelicTeamCallbackValidation: def _data(self, callback_vars): diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index b6eebcb2ef3..6c0695c3bb0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -1674,6 +1674,62 @@ async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller patched_prisma.db.litellm_teamtable.update.assert_awaited_once() +@pytest.mark.asyncio +async def test_a_second_entry_may_not_flip_internal_spans_but_another_backend_may(patched_prisma): + """The var merges per backend, so a second langfuse_otel entry saying include next to + a stored exclude would export whichever is stored last and is refused, while an arize + entry saying include is a different destination and is stored as written.""" + patched_prisma.get_data = AsyncMock( + return_value=_team_row( + metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "otel_internal_spans": "exclude", + }, + } + ] + } + ) + ) + with pytest.raises(HTTPException) as exc: + await add_team_callbacks( + data=AddTeamCallback( + callback_name="langfuse_otel", + callback_type="failure", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "otel_internal_spans": "include", + }, + ), + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + assert exc.value.status_code == 400 + assert "otel_internal_spans" in str(exc.value.detail) and "'exclude'" in str(exc.value.detail) + patched_prisma.db.litellm_teamtable.update.assert_not_called() + + await add_team_callbacks( + data=AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_api_key": "k", "arize_space_id": "s", "otel_internal_spans": "include"}, + ), + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + patched_prisma.db.litellm_teamtable.update.assert_awaited_once() + saved = json.loads(patched_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + assert [entry["callback_vars"]["otel_internal_spans"] for entry in saved["logging"]] == ["exclude", "include"] + + @pytest.mark.asyncio async def test_add_team_callbacks_rejects_out_of_range_arize_sampling_rate(patched_prisma): data = AddTeamCallback( diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index f5138d55b5d..582ddf0987c 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -32,6 +32,10 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ arize_space_id: "password", arize_success_sampling_rate: "number", arize_error_sampling_rate: "number", + otel_internal_spans: "select", + }, + dynamic_param_options: { + otel_internal_spans: ["include", "exclude"], }, description: "Arize Logging Integration", }, @@ -90,6 +94,10 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ dynamic_params: { newrelic_api_key: "password", newrelic_region: "text", + otel_internal_spans: "select", + }, + dynamic_param_options: { + otel_internal_spans: ["include", "exclude"], }, description: "New Relic Logging Integration", }, @@ -128,9 +136,11 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ langfuse_host: "text", langfuse_environment: "text", langfuse_span_scope: "select", + otel_internal_spans: "select", }, dynamic_param_options: { langfuse_span_scope: ["full", "llm_only"], + otel_internal_spans: ["include", "exclude"], }, description: "Langfuse v3 OTEL Logging Integration", }, diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx index 5ce8ff71267..4943687452e 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx @@ -239,6 +239,29 @@ describe("LoggingSettings", () => { ]); }); + it("offers the OTEL destination internal spans as a pick between include and exclude", async () => { + const user = userEvent.setup({ delay: null }); + const mockOnChange = vi.fn(); + const initialValue = [ + { + callback_name: "arize", + callback_type: "success", + callback_vars: {}, + }, + ]; + + renderWithProviders(); + + expect(screen.queryByPlaceholderText("os.environ/OTEL_INTERNAL_SPANS")).not.toBeInTheDocument(); + await user.click(screen.getByRole("combobox", { name: "otel internal spans" })); + expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual(["include", "exclude"]); + await user.click(screen.getByRole("option", { name: "exclude" })); + + expect(mockOnChange).toHaveBeenCalledWith([ + expect.objectContaining({ callback_vars: expect.objectContaining({ otel_internal_spans: "exclude" }) }), + ]); + }); + it("renders sampling rate inputs for the Arize callback and records changes", () => { const mockOnChange = vi.fn(); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index be1e000bd66..726d810047e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16322,6 +16322,7 @@ export interface paths { * - langfuse_host: The host for the Langfuse callback * - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) * - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + * - otel_internal_spans: For langfuse_otel, arize, weave_otel and newrelic, "include" sends the proxy's own SERVICE and DB_CALL spans (auth, Redis, Postgres) with the trace, "exclude" holds them back; defaults to litellm_settings.otel_tenant_internal_spans * - gcs_bucket_name: The name of the GCS bucket * - gcs_path_service_account: The path to the GCS service account * - langsmith_api_key: The API key for the Langsmith callback