fix(otel): stop printing spans to stdout when no exporter is configured

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-10-28 00:55:32 +00:00
parent c17faa1a27
commit 954646e781
14 changed files with 288 additions and 21 deletions

View file

@ -2,11 +2,14 @@ import base64
import json
import os
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from datetime import datetime
from itertools import chain
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
from opentelemetry import trace
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
import litellm
from litellm._logging import verbose_logger
@ -61,6 +64,11 @@ class _Gettable(Protocol):
def get(self, key: str, default: object = None) -> object: ...
class _DiscardingSpanExporter(SpanExporter):
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
return SpanExportResult.SUCCESS
def _as_gettable(value: object) -> _Gettable | None:
return value if isinstance(value, _Gettable) else None
@ -382,8 +390,18 @@ class LangfuseOtelLogger(OpenTelemetry):
secret_key: Final = os.environ.get("LANGFUSE_SECRET_KEY", None)
if not public_key or not secret_key:
# If no keys, return default from env (likely logging to console or something else)
return OpenTelemetryConfig.from_env()
config: Final = OpenTelemetryConfig.from_env()
if any(
name in os.environ
for name in (
"OTEL_EXPORTER",
"OTEL_EXPORTER_OTLP_PROTOCOL",
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_ENDPOINT",
)
):
return config
return replace(config, exporter=_DiscardingSpanExporter())
return LangfuseOtelLogger._build_langfuse_otel_config(
public_key=public_key,

View file

@ -31,7 +31,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.model.baggage import promoted_baggage
from litellm.integrations.otel.model.config import OpenTelemetryV2Config, excluded_db_systems_from
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config, excluded_db_systems_from
from litellm.integrations.otel.model.metadata import (
LLMCallEvent,
RequestIdentity,
@ -83,6 +83,7 @@ from litellm.integrations.otel.plumbing.providers import (
resolve_meter_provider,
)
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
from litellm.integrations.otel.presets.utils import is_unconfigured_placeholder
from litellm.types.utils import captures_span_content
if TYPE_CHECKING:
@ -933,7 +934,15 @@ def select_global_otel_v2_logger(
if registered is not None:
return registered
existing: Final = next((cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)), None)
return existing if existing is not None else OpenTelemetryV2()
if existing is not None:
return existing
fallback_config: Final = OpenTelemetryV2Config()
config: Final = (
fallback_config.model_copy(update={"exporters": (ExporterSpec(requires_headers=True),)})
if all(is_unconfigured_placeholder(spec) for spec in fallback_config.exporters)
else fallback_config
)
return build_otel_v2_logger(config=config)
def publish_global_otel_v2_provider(

View file

@ -25,6 +25,7 @@ from litellm.integrations.otel.model.config import (
OpenTelemetryV2Config,
)
from litellm.integrations.otel.plumbing.providers import register_exporter_factory
from litellm.integrations.otel.presets.utils import operator_exporters
_AGENTOPS_ENDPOINT: Final = "https://otlp.agentops.ai/v1/traces"
_AGENTOPS_AUTH_ENDPOINT: Final = "https://api.agentops.ai/v3/auth/token"
@ -56,7 +57,7 @@ def agentops_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
*operator_exporters(base.exporters),
ExporterSpec(
kind=_AGENTOPS_EXPORTER_KIND,
endpoint=_AGENTOPS_ENDPOINT,

View file

@ -11,7 +11,7 @@ from litellm.integrations.otel.model.config import (
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import credential_gated_exporters, ensure_mappers
from litellm.integrations.otel.presets.utils import credential_gated_exporters, ensure_mappers, operator_exporters
from litellm.types.utils import StandardCallbackDynamicParams
#: Arize routes an export to a project by the ``model_id`` resource attribute and
@ -52,7 +52,7 @@ def arize_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
*operator_exporters(base.exporters),
ExporterSpec(
kind=arize_cfg.protocol or "otlp_grpc",
endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1",

View file

@ -13,6 +13,7 @@ from litellm.integrations.otel.model.config import (
from litellm.integrations.otel.presets.utils import (
credential_gated_exporters,
ensure_mappers,
operator_exporters,
)
from litellm.types.utils import StandardCallbackDynamicParams
@ -39,7 +40,7 @@ def langfuse_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
*operator_exporters(base.exporters),
ExporterSpec(
kind=kind,
endpoint=cfg.endpoint,

View file

@ -8,6 +8,7 @@ from litellm.integrations.otel.model.config import (
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import operator_exporters
def levo_preset(
@ -20,7 +21,7 @@ def levo_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
*operator_exporters(base.exporters),
ExporterSpec(
kind="otlp_http",
endpoint=cfg.endpoint,

View file

@ -14,7 +14,7 @@ from litellm.integrations.otel.model.config import (
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.integrations.otel.presets.utils import ensure_mappers, operator_exporters
from litellm.types.utils import StandardCallbackDynamicParams
#: Region -> OTLP base endpoint. A fixed table by design: team config picks a
@ -55,7 +55,7 @@ def newrelic_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
*operator_exporters(base.exporters),
ExporterSpec(
kind="otlp_http",
endpoint=endpoint,

View file

@ -15,7 +15,7 @@ from litellm.integrations.otel.model.config import (
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.integrations.otel.presets.utils import ensure_mappers, operator_exporters
class _PhoenixSettings(BaseSettings):
@ -69,7 +69,7 @@ def phoenix_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
*operator_exporters(base.exporters),
ExporterSpec(
kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http",
endpoint=cfg.endpoint,

View file

@ -12,7 +12,7 @@ from litellm.integrations.otel.model.config import (
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.integrations.otel.presets.utils import ensure_mappers, operator_exporters
from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host
from litellm.types.utils import StandardCallbackDynamicParams
@ -44,7 +44,7 @@ def signoz_preset(
return base.model_copy(
update=MappingProxyType(
{
"exporters": (*base.exporters, spec),
"exporters": (*operator_exporters(base.exporters), spec),
"mapper_names": ensure_mappers(base.mapper_names, "genai"),
}
)

View file

@ -19,6 +19,10 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
return result
def operator_exporters(exporters: Iterable[ExporterSpec]) -> tuple[ExporterSpec, ...]:
return tuple(spec for spec in exporters if not is_unconfigured_placeholder(spec))
def credential_gated_exporters(
exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner"
) -> "tuple[ExporterSpec, ...]":
@ -32,7 +36,7 @@ def credential_gated_exporters(
override filter still recognises which backend this provider speaks for.
"""
return (
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
*operator_exporters(exporters),
ExporterSpec(owner=owner, requires_headers=True),
)

View file

@ -10,6 +10,7 @@ from litellm.integrations.otel.model.config import (
from litellm.integrations.otel.presets.utils import (
credential_gated_exporters,
ensure_mappers,
operator_exporters,
)
from litellm.integrations.weave.weave_otel import (
get_weave_authorization_header,
@ -39,7 +40,7 @@ def weave_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
*operator_exporters(base.exporters),
ExporterSpec(
kind=weave_cfg.protocol or "otlp_http",
endpoint=weave_cfg.endpoint,

View file

@ -14,6 +14,7 @@ import os
import re
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from io import StringIO
from pathlib import Path
from types import MappingProxyType
from typing import Final
@ -24,6 +25,7 @@ import pytest
pytest.importorskip("opentelemetry")
from opentelemetry import trace # noqa: E402
from opentelemetry.sdk.trace.export import ConsoleSpanExporter # noqa: E402
from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402
InMemorySpanExporter,
)
@ -2699,6 +2701,70 @@ def test_select_global_otel_v2_logger_builds_one_when_none_registered():
assert isinstance(chosen, OpenTelemetryV2)
def _unset_otel_exporter_env(monkeypatch: pytest.MonkeyPatch) -> None:
for name in (
"OTEL_EXPORTER",
"OTEL_EXPORTER_OTLP_PROTOCOL",
"OTEL_ENDPOINT",
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_TRACES_ENDPOINT",
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
"OTEL_HEADERS",
"OTEL_EXPORTER_OTLP_HEADERS",
):
monkeypatch.delenv(name, raising=False)
def _capture_console_output(monkeypatch: pytest.MonkeyPatch) -> StringIO:
output: Final = StringIO()
class _CapturingConsoleSpanExporter(ConsoleSpanExporter):
def __init__(self) -> None:
super().__init__(out=output)
monkeypatch.setattr(providers, "ConsoleSpanExporter", _CapturingConsoleSpanExporter)
return output
def test_select_global_otel_v2_logger_without_exporter_does_not_print(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.integrations.otel.logger import select_global_otel_v2_logger
_unset_otel_exporter_env(monkeypatch)
console_output: Final = _capture_console_output(monkeypatch)
logger: Final = select_global_otel_v2_logger([])
with logger.tracer.start_as_current_span("fallback"):
pass
assert console_output.getvalue() == ""
def test_select_global_otel_v2_logger_keeps_explicit_console_exporter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.integrations.otel.logger import select_global_otel_v2_logger
_unset_otel_exporter_env(monkeypatch)
console_output: Final = _capture_console_output(monkeypatch)
monkeypatch.setenv("OTEL_EXPORTER", "console")
logger: Final = select_global_otel_v2_logger([])
with logger.tracer.start_as_current_span("explicit-console"):
pass
assert "explicit-console" in console_output.getvalue()
def test_select_global_otel_v2_logger_fallback_allows_tenant_fan_out(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.integrations.otel.logger import select_global_otel_v2_logger
_unset_otel_exporter_env(monkeypatch)
logger: Final = select_global_otel_v2_logger([])
providers.attach_tenant_fan_out(logger._tracer_provider, logger.config)
assert any(
isinstance(processor, providers.TenantFanOutSpanProcessor)
for processor in logger._tracer_provider._active_span_processor._span_processors
)
def test_publish_global_otel_v2_provider_sets_selected_logger_provider(monkeypatch):
"""The startup publish must set the OTel global provider to the *selected*
logger's provider (the preset logger that owns every exporter), so the FastAPI

View file

@ -3,15 +3,19 @@ event loop: the preset does no network I/O, and a custom exporter mints the JWT
lazily on its first export (in the BatchSpanProcessor worker thread)."""
import json
from collections.abc import Callable, Mapping
from io import StringIO
from typing import Final
import httpx
import pytest
import requests_mock
import respx
from opentelemetry.sdk.trace.export import SpanExportResult
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SpanExportResult
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing import providers
from litellm.integrations.otel.model.config import ExporterSpec
from litellm.integrations.otel.presets import agentops as agentops_mod
from litellm.integrations.otel.presets.agentops import (
_AGENTOPS_AUTH_ENDPOINT,
@ -21,6 +25,132 @@ from litellm.integrations.otel.presets.agentops import (
_fetch_agentops_jwt,
agentops_preset,
)
from litellm.integrations.otel.presets.arize import arize_preset
from litellm.integrations.otel.presets.langfuse import langfuse_preset
from litellm.integrations.otel.presets.levo import levo_preset
from litellm.integrations.otel.presets.newrelic import newrelic_preset
from litellm.integrations.otel.presets.phoenix import phoenix_preset
from litellm.integrations.otel.presets.signoz import signoz_preset
from litellm.integrations.otel.presets.weave import weave_preset
_PRESET_ENV_VARS: Final = (
"LANGFUSE_PUBLIC_KEY",
"LANGFUSE_SECRET_KEY",
"LANGFUSE_HOST",
"LANGFUSE_OTEL_HOST",
"ARIZE_SPACE_ID",
"ARIZE_SPACE_KEY",
"ARIZE_API_KEY",
"OTEL_EXPORTER_OTLP_TRACES_HEADERS",
"WANDB_API_KEY",
"WANDB_PROJECT_ID",
"LEVOAI_API_KEY",
"LEVOAI_ORG_ID",
"LEVOAI_WORKSPACE_ID",
"LEVOAI_COLLECTOR_URL",
"PHOENIX_API_KEY",
"PHOENIX_COLLECTOR_ENDPOINT",
"PHOENIX_COLLECTOR_PROJECT_NAME",
"SIGNOZ_INGESTION_ENDPOINT",
"SIGNOZ_INGESTION_KEY",
"NEW_RELIC_LICENSE_KEY",
"NEW_RELIC_REGION",
"NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED",
"AGENTOPS_API_KEY",
"AGENTOPS_SERVICE_NAME",
"AGENTOPS_ENVIRONMENT",
"OTEL_EXPORTER",
"OTEL_EXPORTER_OTLP_PROTOCOL",
"OTEL_ENDPOINT",
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_TRACES_ENDPOINT",
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
"OTEL_HEADERS",
"OTEL_EXPORTER_OTLP_HEADERS",
)
_PRESET_CASES: Final = (
pytest.param(
langfuse_preset,
{"LANGFUSE_PUBLIC_KEY": "pk", "LANGFUSE_SECRET_KEY": "sk"},
id="langfuse",
),
pytest.param(arize_preset, {"ARIZE_SPACE_ID": "space", "ARIZE_API_KEY": "key"}, id="arize"),
pytest.param(weave_preset, {"WANDB_API_KEY": "key", "WANDB_PROJECT_ID": "entity/project"}, id="weave"),
pytest.param(
levo_preset,
{
"LEVOAI_API_KEY": "key",
"LEVOAI_ORG_ID": "org",
"LEVOAI_WORKSPACE_ID": "workspace",
"LEVOAI_COLLECTOR_URL": "http://levo.test/v1/traces",
},
id="levo",
),
pytest.param(phoenix_preset, {"PHOENIX_API_KEY": "key"}, id="phoenix"),
pytest.param(
signoz_preset,
{"SIGNOZ_INGESTION_ENDPOINT": "http://signoz.test/v1/traces", "SIGNOZ_INGESTION_KEY": "key"},
id="signoz",
),
pytest.param(newrelic_preset, {"NEW_RELIC_LICENSE_KEY": "key"}, id="newrelic"),
pytest.param(agentops_preset, {"AGENTOPS_API_KEY": "key"}, id="agentops"),
)
def _clear_preset_env(monkeypatch: pytest.MonkeyPatch) -> None:
for name in _PRESET_ENV_VARS:
monkeypatch.delenv(name, raising=False)
def _capture_console_output(monkeypatch: pytest.MonkeyPatch) -> StringIO:
output: Final = StringIO()
class _CapturingConsoleSpanExporter(ConsoleSpanExporter):
def __init__(self) -> None:
super().__init__(out=output)
monkeypatch.setattr(providers, "ConsoleSpanExporter", _CapturingConsoleSpanExporter)
return output
def _provider_for_console_behavior(preset: Callable[[], OpenTelemetryV2Config]) -> TracerProvider:
config: Final = preset()
exporters: Final = tuple(
spec if spec.kind.lower() == "console" else ExporterSpec(kind="in_memory") for spec in config.exporters
)
safe_config: Final = config.model_copy(update={"exporters": exporters})
return providers.build_tracer_provider(safe_config)
@pytest.mark.parametrize(("preset", "credentials"), _PRESET_CASES)
def test_credentialed_presets_do_not_export_to_default_console(
preset: Callable[[], OpenTelemetryV2Config],
credentials: Mapping[str, str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
_clear_preset_env(monkeypatch)
console_output: Final = _capture_console_output(monkeypatch)
for name, value in credentials.items():
monkeypatch.setenv(name, value)
provider: Final = _provider_for_console_behavior(preset)
with provider.get_tracer("litellm").start_as_current_span("preset") as span:
span.set_attribute("preset", "credentialed")
assert console_output.getvalue() == ""
def test_credentialed_preset_keeps_explicit_console_exporter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_clear_preset_env(monkeypatch)
console_output: Final = _capture_console_output(monkeypatch)
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk")
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk")
monkeypatch.setenv("OTEL_EXPORTER", "console")
provider: Final = _provider_for_console_behavior(langfuse_preset)
with provider.get_tracer("litellm").start_as_current_span("explicit-console") as span:
span.set_attribute("preset", "console")
assert "explicit-console" in console_output.getvalue()
def test_agentops_preset_does_no_network_io(monkeypatch):

View file

@ -4,14 +4,17 @@ import os
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from io import StringIO
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
import opentelemetry.sdk.trace.export as trace_export
from openai.types.responses import ResponseFunctionToolCall
from opentelemetry import trace
from opentelemetry.context import Context
from opentelemetry.sdk.trace import SpanLimits, TracerProvider
from opentelemetry.sdk.trace.export import ConsoleSpanExporter
from opentelemetry.trace import Span
import litellm
@ -627,8 +630,6 @@ class TestLangfuseOtelKeyDynamicConfig:
with self._clean_env():
logger = LangfuseOtelLogger()
assert logger.OTEL_EXPORTER == "console"
tracer = logger.get_tracer_to_use_for_request(
{"standard_callback_dynamic_params": self._dynamic_params()}
)
@ -1474,6 +1475,41 @@ def test_langfuse_otel_env_config_headers_carry_v4_ingestion_and_basic_auth() ->
assert _decoded_basic_auth(headers["Authorization"]) == "pk-lf-123:sk-lf-123"
def _capture_console_output(monkeypatch: pytest.MonkeyPatch) -> StringIO:
output: Final = StringIO()
class _CapturingConsoleSpanExporter(ConsoleSpanExporter):
def __init__(self) -> None:
super().__init__(out=output)
monkeypatch.setattr(trace_export, "ConsoleSpanExporter", _CapturingConsoleSpanExporter)
return output
def test_langfuse_otel_without_keys_or_exporter_does_not_print(
monkeypatch: pytest.MonkeyPatch,
) -> None:
with patch.dict(os.environ, LANGFUSE_ENV_ONLY, clear=True):
console_output: Final = _capture_console_output(monkeypatch)
logger: Final = LangfuseOtelLogger(callback_name="langfuse_otel")
with logger.tracer.start_as_current_span("langfuse-default"):
pass
assert logger._tracer_provider.force_flush()
assert console_output.getvalue() == ""
def test_langfuse_otel_without_keys_keeps_explicit_console_exporter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
with patch.dict(os.environ, {**LANGFUSE_ENV_ONLY, "OTEL_EXPORTER": "console"}, clear=True):
console_output: Final = _capture_console_output(monkeypatch)
logger: Final = LangfuseOtelLogger(callback_name="langfuse_otel")
with logger.tracer.start_as_current_span("langfuse-explicit-console"):
pass
assert logger._tracer_provider.force_flush()
assert "langfuse-explicit-console" in console_output.getvalue()
def test_langfuse_otel_dynamic_headers_carry_v4_ingestion_and_basic_auth() -> None:
with patch.dict(os.environ, LANGFUSE_ENV_ONLY, clear=True):
logger: Final = LangfuseOtelLogger()