mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(otel v2): each callback exports only to the destination it owns
Presets (langfuse_otel, arize, phoenix, weave_otel, newrelic, agentops, levo) no longer fold the operator's OTEL_* exporter into their own exporter list, so vendor-mapped spans reach only the vendor backend. The otel callback alone serves OTEL_*, is the only logger a second otel entry reuses, and takes the proxy global slot over a preset whatever the callback order. The Langfuse logger stamps trace name and content on the request root only when its own provider created that root. The langfuse.* attribute filter is removed since no exporter can receive another callback's vocabulary any more. Credentialless presets keep only their header-gated exporter instead of inheriting the console placeholder, and the v2 Weave preset reads its config without publishing it as process-wide OTEL_* env Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
4a075423a9
commit
210cbf36e9
16 changed files with 330 additions and 356 deletions
|
|
@ -1,6 +1,9 @@
|
|||
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from opentelemetry.sdk.trace import ReadableSpan
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.mappers.langfuse import (
|
||||
|
|
@ -19,15 +22,22 @@ if TYPE_CHECKING:
|
|||
|
||||
class LangfuseOpenTelemetryV2(OpenTelemetryV2):
|
||||
"""Names the trace from the request. Langfuse reads ``langfuse.trace.name`` off the root observation,
|
||||
and the proxy's root span is still recording when the LLM call starts."""
|
||||
and the proxy's root span is still recording when the LLM call starts. Only a root created by this
|
||||
logger's own provider is stamped; any other root is exported to destinations that are not Langfuse."""
|
||||
|
||||
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
|
||||
root: Final = request_root_span()
|
||||
root: Final = self._owned_recording_root()
|
||||
name: Final = caller_trace_name(kwargs)
|
||||
if root is not None and root.is_recording() and name is not None:
|
||||
if root is not None and name is not None:
|
||||
root.set_attribute(LANGFUSE_TRACE_NAME, name)
|
||||
super().log_pre_api_call(model, messages, kwargs)
|
||||
|
||||
def _owned_recording_root(self) -> Span | None:
|
||||
root: Final = request_root_span()
|
||||
if root is None or not root.is_recording() or not isinstance(root, ReadableSpan):
|
||||
return None
|
||||
return root if root.resource is self.tracer_provider.resource else None
|
||||
|
||||
|
||||
class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2):
|
||||
"""Stamps the request's input and output on the root observation while it is still recording.
|
||||
|
|
@ -59,8 +69,8 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2):
|
|||
self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data))
|
||||
|
||||
def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None:
|
||||
root: Final = request_root_span()
|
||||
if root is None or not root.is_recording():
|
||||
root: Final = self._owned_recording_root()
|
||||
if root is None:
|
||||
return
|
||||
try:
|
||||
output: Final = render_output()
|
||||
|
|
|
|||
|
|
@ -259,9 +259,15 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self._register_in_callback_list(litellm._async_failure_callback)
|
||||
except Exception:
|
||||
pass
|
||||
if getattr(proxy_server, "open_telemetry_logger", None) is None:
|
||||
if self._outranks_for_proxy_slot(getattr(proxy_server, "open_telemetry_logger", None)):
|
||||
setattr(proxy_server, "open_telemetry_logger", self)
|
||||
|
||||
def _outranks_for_proxy_slot(self, holder: object) -> bool:
|
||||
"""``otel`` owns the slot whenever configured; a preset holds it only until ``otel`` is built."""
|
||||
if holder is None:
|
||||
return True
|
||||
return self.callback_name is None and isinstance(holder, OpenTelemetryV2) and holder.callback_name is not None
|
||||
|
||||
# ====================================================================== #
|
||||
# LLM-call callbacks — the span is opened at the ``pre_call`` boundary and
|
||||
# closed here. See ``log_pre_api_call``.
|
||||
|
|
@ -842,10 +848,12 @@ def select_global_otel_v2_logger(
|
|||
``proxy_server.open_telemetry_logger``), and every other v2 entry point —
|
||||
guardrail, identity seeding, phase spans — already routes through that same
|
||||
``registered`` owner. Reuse it here too so the global provider has one source
|
||||
of truth instead of a second, independently-derived guess; this is the logger
|
||||
a preset (arize, langfuse, …) folds the ``OTEL_*`` base exporter and its own
|
||||
exporter into, so the FastAPI server span and the gen-ai spans share one
|
||||
provider and one trace.
|
||||
of truth instead of a second, independently-derived guess. Each logger exports
|
||||
only to the destination its callback owns (``otel`` serves ``OTEL_*``, a preset
|
||||
serves its own backend), so the FastAPI server span and the request root land
|
||||
at the ``otel`` callback's collector when one is configured and at the first
|
||||
preset's backend otherwise, whatever the callback order; a preset stamps that
|
||||
root only when its own provider created it.
|
||||
|
||||
Fall back to ``in_memory_loggers`` for the SDK path, where no proxy global is
|
||||
set (selecting from there, not ``service_callback``, which a preset logger does
|
||||
|
|
@ -855,8 +863,11 @@ def select_global_otel_v2_logger(
|
|||
"""
|
||||
if registered is not None:
|
||||
return registered
|
||||
existing: Final = next((cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)), None)
|
||||
return existing if existing is not None else OpenTelemetryV2()
|
||||
v2_loggers: Final = tuple(cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2))
|
||||
generic: Final = next((cb for cb in v2_loggers if cb.callback_name is None), None)
|
||||
if generic is not None:
|
||||
return generic
|
||||
return v2_loggers[0] if v2_loggers else OpenTelemetryV2()
|
||||
|
||||
|
||||
def publish_global_otel_v2_provider(
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ from opentelemetry.util.types import Attributes, AttributeValue
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._version import version as litellm_version
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.semconv import (
|
||||
DB,
|
||||
MCP,
|
||||
|
|
@ -377,7 +377,6 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade
|
|||
# ``?key=`` query parameter.
|
||||
_URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"})
|
||||
_URL_QUERY_KEY: Final = "url.query"
|
||||
_LANGFUSE_ATTRIBUTE_PREFIX: Final = "langfuse."
|
||||
|
||||
|
||||
class _TenantSpanView(ReadableSpan):
|
||||
|
|
@ -419,11 +418,9 @@ def _guardrail_unreachable(attributes: Mapping[str, AttributeValue]) -> bool:
|
|||
return attributes.get(LiteLLM.GUARDRAIL_STATUS) in _GUARDRAIL_UNREACHABLE_STATUSES
|
||||
|
||||
|
||||
def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool, langfuse: bool) -> bool:
|
||||
def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool) -> bool:
|
||||
if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY):
|
||||
return False
|
||||
if not langfuse and key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX):
|
||||
return False
|
||||
if database and key in _DATASTORE_ENDPOINT_KEYS:
|
||||
return False
|
||||
if unreachable_guardrail and key == LiteLLM.GUARDRAIL_RESPONSE:
|
||||
|
|
@ -473,12 +470,11 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read
|
|||
database: Final = _is_database_span(attributes)
|
||||
owned: Final = _is_tenant_owned_span(attributes)
|
||||
unreachable: Final = _guardrail_unreachable(attributes)
|
||||
langfuse: Final = destination.callback_name == ExporterOwner.LANGFUSE_OTEL.value
|
||||
kept: Final = MappingProxyType(
|
||||
{
|
||||
key: _without_query(key, value)
|
||||
for key, value in attributes.items()
|
||||
if _tenant_visible(key, database, owned, unreachable, langfuse)
|
||||
if _tenant_visible(key, database, owned, unreachable)
|
||||
}
|
||||
)
|
||||
recorded: Final = span.events
|
||||
|
|
@ -491,24 +487,6 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read
|
|||
return _TenantSpanView(span, resource, kept, events, status)
|
||||
|
||||
|
||||
def _without_langfuse_attributes(span: ReadableSpan) -> ReadableSpan:
|
||||
attributes: Final = span.attributes or _NO_ATTRIBUTES
|
||||
kept: Final = MappingProxyType(
|
||||
{key: value for key, value in attributes.items() if not key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX)}
|
||||
)
|
||||
if len(kept) == len(attributes):
|
||||
return span
|
||||
return _TenantSpanView(span, span.resource, kept, span.events, span.status)
|
||||
|
||||
|
||||
def _reads_langfuse_attributes(spec: ExporterSpec, config: OpenTelemetryV2Config) -> bool:
|
||||
if spec.owner is not None:
|
||||
return spec.owner is ExporterOwner.LANGFUSE_OTEL
|
||||
return "langfuse" in config.mapper_names and all(
|
||||
other.owner is not ExporterOwner.LANGFUSE_OTEL for other in config.exporters
|
||||
)
|
||||
|
||||
|
||||
class TenantFanOutSpanProcessor(SpanProcessor):
|
||||
"""Export every finished span to each destination this request resolved.
|
||||
|
||||
|
|
@ -796,23 +774,6 @@ class _OverriddenBackendFilter(SpanProcessor):
|
|||
return self._inner.force_flush(timeout_millis)
|
||||
|
||||
|
||||
class _LangfuseAttributeFilter(SpanProcessor):
|
||||
def __init__(self, inner: SpanProcessor) -> None:
|
||||
self._inner: Final = inner
|
||||
|
||||
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
|
||||
self._inner.on_start(span, parent_context)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
self._inner.on_end(_without_langfuse_attributes(span))
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._inner.shutdown()
|
||||
|
||||
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
||||
return self._inner.force_flush(timeout_millis)
|
||||
|
||||
|
||||
def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
|
||||
"""Build a single exporter from the top-level config fields.
|
||||
|
||||
|
|
@ -1099,10 +1060,9 @@ def build_tracer_provider(
|
|||
exp,
|
||||
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
|
||||
)
|
||||
sink = processor if _reads_langfuse_attributes(spec, config) else _LangfuseAttributeFilter(processor)
|
||||
owner = spec.owner.value if spec.owner is not None else None
|
||||
provider.add_span_processor(
|
||||
_OverriddenBackendFilter(sink, owner) if tenant_overrides and owner is not None else sink
|
||||
_OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor
|
||||
)
|
||||
return provider
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,6 @@ def agentops_preset(
|
|||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=_AGENTOPS_EXPORTER_KIND,
|
||||
endpoint=_AGENTOPS_ENDPOINT,
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ def arize_preset(
|
|||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=arize_cfg.protocol or "otlp_grpc",
|
||||
endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1",
|
||||
|
|
|
|||
|
|
@ -10,10 +10,7 @@ from litellm.integrations.otel.model.config import (
|
|||
ExporterSpec,
|
||||
OpenTelemetryV2Config,
|
||||
)
|
||||
from litellm.integrations.otel.presets.utils import (
|
||||
credential_gated_exporters,
|
||||
ensure_mappers,
|
||||
)
|
||||
from litellm.integrations.otel.presets.utils import ensure_mappers
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
||||
|
|
@ -31,7 +28,7 @@ def langfuse_preset(
|
|||
raise
|
||||
return base.model_copy(
|
||||
update={ # mutable-ok: pydantic model_copy takes a plain update mapping
|
||||
"exporters": credential_gated_exporters(base.exporters, ExporterOwner.LANGFUSE_OTEL),
|
||||
"exporters": [ExporterSpec(owner=ExporterOwner.LANGFUSE_OTEL, requires_headers=True)],
|
||||
"mapper_names": mappers,
|
||||
}
|
||||
)
|
||||
|
|
@ -39,7 +36,6 @@ def langfuse_preset(
|
|||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=kind,
|
||||
endpoint=cfg.endpoint,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ def levo_preset(
|
|||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind="otlp_http",
|
||||
endpoint=cfg.endpoint,
|
||||
|
|
|
|||
|
|
@ -54,7 +54,6 @@ def newrelic_preset(
|
|||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind="otlp_http",
|
||||
endpoint=endpoint,
|
||||
|
|
|
|||
|
|
@ -69,7 +69,6 @@ def phoenix_preset(
|
|||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http",
|
||||
endpoint=cfg.endpoint,
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@
|
|||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
|
||||
|
||||
|
||||
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
|
||||
"""Return ``mapper_names`` with each of ``names`` appended if not already present.
|
||||
|
|
@ -17,32 +15,3 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
|
|||
if name not in result:
|
||||
result.append(name)
|
||||
return result
|
||||
|
||||
|
||||
def credential_gated_exporters(
|
||||
exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner"
|
||||
) -> "tuple[ExporterSpec, ...]":
|
||||
"""``exporters`` with the operator's destination replaced by a header-gated one.
|
||||
|
||||
Used when a credential-mandatory backend is asked to build without the operator's
|
||||
own credentials, so only key/team destinations receive spans. Two things have to
|
||||
happen for that to mean "export nowhere": the placeholder console spec that
|
||||
``OpenTelemetryV2Config`` folds in for an empty exporter list is dropped, or every
|
||||
span would be printed to stdout, and the gated spec keeps the owner so the
|
||||
override filter still recognises which backend this provider speaks for.
|
||||
"""
|
||||
return (
|
||||
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
|
||||
ExporterSpec(owner=owner, requires_headers=True),
|
||||
)
|
||||
|
||||
|
||||
def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
|
||||
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
|
||||
|
||||
No field set is what says the operator asked for nothing: an exporter they did
|
||||
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
|
||||
and so does the gated spec this module appends, which would otherwise eat itself
|
||||
when one preset layers onto another.
|
||||
"""
|
||||
return not spec.model_fields_set
|
||||
|
|
|
|||
|
|
@ -7,13 +7,10 @@ from litellm.integrations.otel.model.config import (
|
|||
ExporterSpec,
|
||||
OpenTelemetryV2Config,
|
||||
)
|
||||
from litellm.integrations.otel.presets.utils import (
|
||||
credential_gated_exporters,
|
||||
ensure_mappers,
|
||||
)
|
||||
from litellm.integrations.otel.presets.utils import ensure_mappers
|
||||
from litellm.integrations.weave.weave_otel import (
|
||||
_get_weave_authorization_header,
|
||||
get_weave_otel_config,
|
||||
read_weave_otel_config,
|
||||
)
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
|
@ -26,20 +23,19 @@ def weave_preset(
|
|||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
mappers: Final = ensure_mappers(base.mapper_names, "openinference", "weave")
|
||||
try:
|
||||
weave_cfg: Final = get_weave_otel_config()
|
||||
weave_cfg: Final = read_weave_otel_config()
|
||||
except Exception:
|
||||
if not allow_missing_credentials:
|
||||
raise
|
||||
return base.model_copy(
|
||||
update={ # mutable-ok: pydantic model_copy takes a plain update mapping
|
||||
"exporters": credential_gated_exporters(base.exporters, ExporterOwner.WEAVE_OTEL),
|
||||
"exporters": [ExporterSpec(owner=ExporterOwner.WEAVE_OTEL, requires_headers=True)],
|
||||
"mapper_names": mappers,
|
||||
}
|
||||
)
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=weave_cfg.protocol or "otlp_http",
|
||||
endpoint=weave_cfg.endpoint,
|
||||
|
|
|
|||
|
|
@ -125,17 +125,8 @@ def weave_otel_endpoint(host: str | None) -> str:
|
|||
return normalized.rstrip("/") + WEAVE_OTEL_ENDPOINT
|
||||
|
||||
|
||||
def get_weave_otel_config() -> WeaveOtelConfig:
|
||||
"""
|
||||
Retrieves the Weave OpenTelemetry configuration based on environment variables.
|
||||
|
||||
Environment Variables:
|
||||
WANDB_API_KEY: Required. W&B API key for authentication.
|
||||
WANDB_PROJECT_ID: Required. Project ID in format <entity>/<project_name>.
|
||||
WANDB_HOST: Optional. Custom Weave host URL. Defaults to cloud endpoint.
|
||||
|
||||
Returns:
|
||||
WeaveOtelConfig: A Pydantic model containing Weave OTEL configuration.
|
||||
def read_weave_otel_config() -> WeaveOtelConfig:
|
||||
"""Weave OTLP settings from ``WANDB_API_KEY``, ``WANDB_PROJECT_ID`` and optional ``WANDB_HOST``.
|
||||
|
||||
Raises:
|
||||
ValueError: If required environment variables are missing.
|
||||
|
|
@ -158,10 +149,6 @@ def get_weave_otel_config() -> WeaveOtelConfig:
|
|||
auth_header: Final = _get_weave_authorization_header(api_key=api_key)
|
||||
otlp_auth_headers: Final = f"Authorization={auth_header},project_id={project_id}"
|
||||
|
||||
# Set standard OTEL environment variables
|
||||
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
|
||||
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
|
||||
|
||||
return WeaveOtelConfig(
|
||||
otlp_auth_headers=otlp_auth_headers,
|
||||
endpoint=endpoint,
|
||||
|
|
@ -170,6 +157,14 @@ def get_weave_otel_config() -> WeaveOtelConfig:
|
|||
)
|
||||
|
||||
|
||||
def get_weave_otel_config() -> WeaveOtelConfig:
|
||||
"""``read_weave_otel_config`` plus the v1 side effect of publishing it as the process-wide ``OTEL_*`` env."""
|
||||
config: Final = read_weave_otel_config()
|
||||
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = config.endpoint
|
||||
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = config.otlp_auth_headers
|
||||
return config
|
||||
|
||||
|
||||
def set_weave_otel_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
|
||||
"""
|
||||
Sets OpenTelemetry span attributes for Weave observability.
|
||||
|
|
|
|||
|
|
@ -4499,7 +4499,7 @@ def _init_custom_logger_compatible_class(
|
|||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, OpenTelemetryV2):
|
||||
if isinstance(callback, OpenTelemetryV2) and callback.callback_name is None:
|
||||
return callback
|
||||
otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
|
||||
otel_logger_v2: Final = build_otel_v2_logger(
|
||||
|
|
@ -4862,13 +4862,13 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
|
|||
Returns ``None`` when V2 is off OR when there's no preset registered for
|
||||
``callback_name`` — callers should then fall through to the legacy path.
|
||||
|
||||
A preset that needs operator credentials it cannot find is allowed to build
|
||||
only when this request has a key/team destination for that backend and another
|
||||
V2 logger is already registered to carry the fan-out. The resulting logger keeps
|
||||
only its credential-gated exporter, while the registered logger owns operator
|
||||
delivery. Without that carrier, a preset that raises or that ends up with nothing
|
||||
but its gated exporter and the default console placeholder returns ``None``, so the
|
||||
caller falls through to the legacy path exactly as before V2 landed.
|
||||
Each preset exports only to the destination it owns; the operator's ``OTEL_*``
|
||||
collector is served by the ``otel`` callback alone. A preset that needs operator
|
||||
credentials it cannot find is allowed to build only when this request has a
|
||||
key/team destination for that backend and another V2 logger is already registered
|
||||
to carry the fan-out. Without that carrier, a preset that raises or that has nothing
|
||||
but its credential-gated exporter returns ``None``, so the caller falls through to
|
||||
the legacy path exactly as before V2 landed.
|
||||
"""
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
|
||||
|
|
@ -4887,21 +4887,20 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
|
|||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
isinstance(callback, OpenTelemetryV2)
|
||||
and getattr(callback, "callback_name", None) == callback_name
|
||||
and callback.callback_name == callback_name
|
||||
and (serves_a_destination or not _exports_nowhere(callback.config))
|
||||
):
|
||||
return callback
|
||||
try:
|
||||
built: Final = preset_fn(allow_missing_credentials=carried)
|
||||
config: Final = preset_fn(allow_missing_credentials=carried)
|
||||
except Exception:
|
||||
# If env vars are missing or the preset raises, defer to the legacy path
|
||||
# so customers get the same error story they had before V2 landed.
|
||||
return None
|
||||
gated: Final = _is_credential_gated(built)
|
||||
if gated and not carried and not _has_operator_exporter(built):
|
||||
exports_nowhere: Final = _exports_nowhere(config)
|
||||
if exports_nowhere and not carried:
|
||||
return None
|
||||
config: Final = _only_the_gated_exporter(built) if gated and carried else built
|
||||
if _exports_nowhere(config):
|
||||
if exports_nowhere:
|
||||
verbose_logger.warning(
|
||||
"OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces",
|
||||
callback_name,
|
||||
|
|
@ -4916,24 +4915,6 @@ def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool:
|
|||
return all(_is_gated(spec) for spec in config.exporters)
|
||||
|
||||
|
||||
def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool:
|
||||
"""Whether the preset built without the operator's own credentials for its backend."""
|
||||
return any(_is_gated(spec) for spec in config.exporters)
|
||||
|
||||
|
||||
def _has_operator_exporter(config: "OpenTelemetryV2Config") -> bool:
|
||||
"""Whether the operator configured somewhere real to export, beyond the default console placeholder."""
|
||||
from litellm.integrations.otel.presets.utils import is_unconfigured_placeholder
|
||||
|
||||
return any(not _is_gated(spec) and not is_unconfigured_placeholder(spec) for spec in config.exporters)
|
||||
|
||||
|
||||
def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config":
|
||||
return config.model_copy(
|
||||
update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update
|
||||
)
|
||||
|
||||
|
||||
def _is_gated(spec: "ExporterSpec") -> bool:
|
||||
return spec.requires_headers and not spec.headers
|
||||
|
||||
|
|
|
|||
|
|
@ -374,6 +374,53 @@ def test_unnamed_request_leaves_the_trace_name_off_both_spans():
|
|||
assert TRACE_NAME_ATTR not in root_attrs and TRACE_NAME_ATTR not in generation_attrs
|
||||
|
||||
|
||||
def test_a_root_owned_by_another_loggers_provider_is_never_stamped_with_langfuse_vocabulary():
|
||||
"""``callbacks: [otel, langfuse_otel]``: the request root belongs to the ``otel`` logger and exports to the
|
||||
operator's collector, so Langfuse's trace name and root input/output must not be written on it."""
|
||||
generic, generic_exporter = _logger(mappers=("genai",))
|
||||
langfuse, langfuse_exporter = _logger()
|
||||
response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))])
|
||||
named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}}
|
||||
|
||||
root = _start_root(generic)
|
||||
asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion"))
|
||||
langfuse.log_pre_api_call(
|
||||
model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named}
|
||||
)
|
||||
asyncio.run(
|
||||
langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response)
|
||||
)
|
||||
root.end()
|
||||
|
||||
root_attrs = _root_attrs(generic_exporter)
|
||||
assert not any(key.startswith("langfuse.") for key in root_attrs)
|
||||
assert all(span.name != LITELLM_PROXY_REQUEST_SPAN_NAME for span in langfuse_exporter.get_finished_spans())
|
||||
|
||||
|
||||
def test_a_root_owned_by_the_langfuse_provider_is_still_named_and_given_its_io_next_to_another_logger():
|
||||
"""``callbacks: [langfuse_otel]`` with a second v2 logger alongside: the root was created by the Langfuse
|
||||
provider, so the ownership check must not stop the trace name and root input/output from reaching Langfuse."""
|
||||
_logger(mappers=("genai",))
|
||||
langfuse, langfuse_exporter = _logger()
|
||||
response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))])
|
||||
named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}}
|
||||
|
||||
root = _start_root(langfuse)
|
||||
asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion"))
|
||||
langfuse.log_pre_api_call(
|
||||
model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named}
|
||||
)
|
||||
asyncio.run(
|
||||
langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response)
|
||||
)
|
||||
root.end()
|
||||
|
||||
root_attrs = _root_attrs(langfuse_exporter)
|
||||
assert root_attrs[TRACE_NAME_ATTR] == "private-name"
|
||||
assert json.loads(root_attrs[INPUT_ATTR]) == CHAT_DATA["messages"]
|
||||
assert "pong" in root_attrs[OUTPUT_ATTR]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("capture", "mappers"),
|
||||
[("no_content", ("genai", "langfuse")), ("span_only", ("genai",))],
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@
|
|||
import contextvars
|
||||
import time
|
||||
from base64 import b64encode
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import reduce
|
||||
from types import MappingProxyType
|
||||
|
||||
import pytest
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
|
|
@ -22,6 +22,7 @@ from litellm.integrations.otel.logger import (
|
|||
build_otel_v2_logger,
|
||||
fan_out_provider,
|
||||
publish_global_otel_v2_provider,
|
||||
select_global_otel_v2_logger,
|
||||
)
|
||||
from litellm.integrations.otel.model.config import (
|
||||
ExporterOwner,
|
||||
|
|
@ -51,6 +52,11 @@ from litellm.integrations.otel.presets.destinations import (
|
|||
destination_for,
|
||||
)
|
||||
from litellm.integrations.otel.presets.langfuse import langfuse_preset
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_init_custom_logger_compatible_class,
|
||||
_maybe_construct_otel_v2,
|
||||
)
|
||||
from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
convert_key_logging_metadata_to_callback,
|
||||
|
|
@ -92,6 +98,12 @@ def emit(provider: TracerProvider, name: str = "chat gpt-4") -> None:
|
|||
pass
|
||||
|
||||
|
||||
def swap_exporter_factories(monkeypatch, **factories: Callable[[ExporterSpec], SpanExporter]) -> None:
|
||||
"""Swap the exporter built for each ``kind`` for the duration of one test."""
|
||||
for kind, factory in factories.items():
|
||||
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, factory)
|
||||
|
||||
|
||||
def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemorySpanExporter) -> TracerProvider:
|
||||
"""The operator's provider: one owned exporter plus the tenant fan-out."""
|
||||
provider = TracerProvider()
|
||||
|
|
@ -516,34 +528,6 @@ class TestFanOut:
|
|||
assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"]
|
||||
assert [s.name for s in arize.get_finished_spans()] == ["chat gpt-4"]
|
||||
|
||||
def test_only_the_langfuse_destination_receives_the_langfuse_attributes(self):
|
||||
langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter()
|
||||
by_endpoint = {"http://a.local": langfuse, "http://b.local": arize}
|
||||
provider = TracerProvider()
|
||||
provider.add_span_processor(
|
||||
TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint]))
|
||||
)
|
||||
|
||||
def run():
|
||||
set_request_destinations(
|
||||
(
|
||||
OtelDestination(endpoint="http://a.local", callback_name="langfuse_otel"),
|
||||
OtelDestination(endpoint="http://b.local", callback_name="arize"),
|
||||
)
|
||||
)
|
||||
span = get_tracer(provider, "litellm").start_span("chat gpt-4")
|
||||
span.set_attribute("gen_ai.request.model", "gpt-4")
|
||||
span.set_attribute("langfuse.trace.name", "private-langfuse-only-name")
|
||||
span.end()
|
||||
|
||||
in_fresh_context(run)
|
||||
|
||||
(langfuse_span,) = langfuse.get_finished_spans()
|
||||
(arize_span,) = arize.get_finished_spans()
|
||||
assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
|
||||
assert arize_span.attributes["gen_ai.request.model"] == "gpt-4"
|
||||
assert [k for k in arize_span.attributes if k.startswith("langfuse.")] == []
|
||||
|
||||
def test_a_destination_carries_the_tenants_service_name(self):
|
||||
"""An overridden backend skips per-request tracer routing, so the service name
|
||||
that route used to apply has to travel on the destination instead."""
|
||||
|
|
@ -1473,8 +1457,6 @@ class TestPresetDegradation:
|
|||
langfuse_preset()
|
||||
|
||||
def test_a_credential_less_proxy_builds_the_gated_logger_beside_a_v2_carrier(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
carrier = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))
|
||||
|
|
@ -1494,8 +1476,6 @@ class TestPresetDegradation:
|
|||
"""Nothing can use a credential-less langfuse here, so the operator has to get
|
||||
the same story as before v2: the legacy integration, not a global provider
|
||||
that exports nowhere."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
||||
|
|
@ -1505,9 +1485,10 @@ class TestPresetDegradation:
|
|||
|
||||
assert logger is None
|
||||
|
||||
def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
def test_a_credentialless_newrelic_does_not_take_over_the_operator_collector(self, monkeypatch):
|
||||
"""The ``OTEL_*`` collector belongs to the ``otel`` callback. A New Relic
|
||||
entry with no license key has nowhere of its own to export, so it takes
|
||||
the legacy path instead of shipping New Relic-shaped spans to the collector."""
|
||||
monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False)
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
|
@ -1516,15 +1497,21 @@ class TestPresetDegradation:
|
|||
logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", [])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert logger is None
|
||||
|
||||
def test_a_credentialed_newrelic_exports_only_to_new_relic(self, monkeypatch):
|
||||
monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "nr-license")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", [])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert logger is not None
|
||||
assert [spec.endpoint for spec in logger.config.exporters] == [
|
||||
"http://collector.local:4318",
|
||||
"https://otlp.nr-data.net",
|
||||
]
|
||||
assert [spec.endpoint for spec in logger.config.exporters] == ["https://otlp.nr-data.net"]
|
||||
|
||||
def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False)
|
||||
for name in _OTEL_SHORTHAND_ENV:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
|
@ -1536,12 +1523,10 @@ class TestPresetDegradation:
|
|||
|
||||
assert logger is None
|
||||
|
||||
def test_an_explicit_console_exporter_keeps_a_credentialless_preset_on_v2(self, monkeypatch, capfd):
|
||||
"""``OTEL_EXPORTER=console`` reads exactly like the placeholder ``_normalize``
|
||||
folds in, but the operator asked for it, so a credential-less New Relic keeps
|
||||
the V2 logger and its spans reach stdout instead of the legacy path."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
def test_an_explicit_console_exporter_is_served_by_otel_not_by_a_credentialless_preset(self, monkeypatch):
|
||||
"""``OTEL_EXPORTER=console`` is the operator's generic destination. Only the
|
||||
``otel`` callback prints there; a credential-less New Relic has nothing of
|
||||
its own to export to and takes the legacy path."""
|
||||
monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False)
|
||||
for name in _OTEL_SHORTHAND_ENV:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
|
@ -1550,15 +1535,13 @@ class TestPresetDegradation:
|
|||
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", [])
|
||||
generic = build_otel_v2_logger(OpenTelemetryV2Config())
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert logger is not None
|
||||
assert logger.config.exporters[0].kind == "console"
|
||||
assert not logger.config.exporters[0].requires_headers
|
||||
assert logger is None
|
||||
assert [spec.kind for spec in generic.config.exporters] == ["console"]
|
||||
|
||||
def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.delenv("WANDB_API_KEY", raising=False)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
|
@ -1576,8 +1559,6 @@ class TestPresetDegradation:
|
|||
def test_the_exporter_less_logger_is_not_reused_by_a_request_without_destinations(self, monkeypatch):
|
||||
"""Reusing it would let one team's destination decide how every later request
|
||||
without one is logged, long after the degrade was justified."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))]
|
||||
|
|
@ -1595,8 +1576,6 @@ class TestPresetDegradation:
|
|||
assert plain is None
|
||||
|
||||
def test_a_credentialed_logger_is_still_reused_across_requests(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1")
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
|
@ -1612,8 +1591,6 @@ class TestPresetDegradation:
|
|||
|
||||
@staticmethod
|
||||
def _degraded_langfuse_beside(loggers, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
|
@ -1643,8 +1620,6 @@ class TestPresetDegradation:
|
|||
"""Only a V2 logger publishes the provider the fan-out rides on, so a legacy
|
||||
callback beside this one leaves the destination just as unreachable as no
|
||||
callback at all, and the operator keeps the pre-V2 story."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
||||
|
|
@ -1658,28 +1633,24 @@ class TestPresetDegradation:
|
|||
|
||||
assert logger is None
|
||||
|
||||
def test_a_credentialed_logger_beside_another_v2_logger_keeps_every_exporter(self, monkeypatch):
|
||||
"""Only a degraded preset gives the collector up; an operator who configured
|
||||
both the backend and the collector still exports to both, as on base."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
@pytest.mark.parametrize("beside", [(), ("collector",)])
|
||||
def test_a_credentialed_langfuse_exports_only_to_langfuse(self, monkeypatch, beside):
|
||||
"""The operator's collector is the ``otel`` callback's destination whether or
|
||||
not that callback is registered; Langfuse never inherits it."""
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1")
|
||||
monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))
|
||||
loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) for _ in beside]
|
||||
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", [collector_logger])
|
||||
logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert logger is not None
|
||||
assert [spec.endpoint for spec in logger.config.exporters] == [
|
||||
"http://collector.local:4318",
|
||||
"https://cloud.langfuse.com/api/public/otel",
|
||||
]
|
||||
assert all(spec.headers for spec in logger.config.exporters if spec.requires_headers)
|
||||
assert [spec.endpoint for spec in logger.config.exporters] == ["https://cloud.langfuse.com/api/public/otel"]
|
||||
assert all(spec.headers for spec in logger.config.exporters)
|
||||
|
||||
|
||||
class TestContextIsolation:
|
||||
|
|
@ -1692,26 +1663,184 @@ class TestContextIsolation:
|
|||
assert in_fresh_context(request_destinations) == ()
|
||||
|
||||
|
||||
class TestOperatorShorthandSurvivesDegradation:
|
||||
def test_a_generic_otlp_collector_keeps_receiving_when_langfuse_has_no_credentials(self, monkeypatch):
|
||||
"""Only the stdout placeholder is dropped. An operator who set the standard
|
||||
OTLP env vars configured a real destination and must keep it."""
|
||||
class TestOperatorShorthandStaysWithOtel:
|
||||
def test_a_credential_less_langfuse_leaves_the_operator_collector_to_the_otel_callback(self, monkeypatch):
|
||||
monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False)
|
||||
monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False)
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
|
||||
|
||||
config = langfuse_preset(allow_missing_credentials=True)
|
||||
|
||||
assert [spec.endpoint for spec in config.exporters] == ["http://collector.local:4318", None]
|
||||
assert [spec.kind for spec in config.exporters] == ["otlp_http", "console"]
|
||||
assert [spec.endpoint for spec in config.exporters] == [None]
|
||||
assert [spec.owner for spec in config.exporters] == [ExporterOwner.LANGFUSE_OTEL]
|
||||
|
||||
def test_the_stdout_placeholder_is_still_dropped_when_it_is_the_only_exporter(self, monkeypatch):
|
||||
def test_the_stdout_placeholder_is_never_inherited_by_a_credential_less_preset(self, monkeypatch):
|
||||
credential_less_proxy(monkeypatch)
|
||||
|
||||
config = langfuse_preset(allow_missing_credentials=True)
|
||||
|
||||
assert all(spec.requires_headers and not spec.headers for spec in config.exporters)
|
||||
|
||||
def test_a_lone_credentialed_preset_without_otel_env_prints_nothing_to_stdout(self, monkeypatch):
|
||||
"""``callbacks: [langfuse_otel]`` and no ``OTEL_*`` at all: the base config's stdout placeholder
|
||||
belongs to ``otel``, so the Langfuse logger must not dump every span as JSON on the proxy log."""
|
||||
for name in _OTEL_SHORTHAND_ENV:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
for name, value in _VENDOR_ENV["langfuse_otel"].items():
|
||||
monkeypatch.setenv(name, value)
|
||||
langfuse_sink, stdout = InMemorySpanExporter(), InMemorySpanExporter()
|
||||
swap_exporter_factories(monkeypatch, otlp_http=lambda spec: langfuse_sink, console=lambda spec: stdout)
|
||||
|
||||
logger = build_otel_v2_logger(langfuse_preset(), callback_name="langfuse_otel")
|
||||
emit(logger.tracer_provider, "vendor only")
|
||||
logger.tracer_provider.force_flush()
|
||||
|
||||
assert [s.name for s in langfuse_sink.get_finished_spans()] == ["vendor only"]
|
||||
assert stdout.get_finished_spans() == ()
|
||||
|
||||
|
||||
_COLLECTOR = "http://collector.local:4318"
|
||||
|
||||
#: Env that gives each endpoint-owning preset its own credentials.
|
||||
_VENDOR_ENV = MappingProxyType(
|
||||
{
|
||||
"langfuse_otel": {
|
||||
"LANGFUSE_PUBLIC_KEY": "pk",
|
||||
"LANGFUSE_SECRET_KEY": "sk",
|
||||
"LANGFUSE_HOST": "https://lf.local",
|
||||
},
|
||||
"arize": {"ARIZE_SPACE_ID": "space", "ARIZE_API_KEY": "key"},
|
||||
"arize_phoenix": {
|
||||
"PHOENIX_API_KEY": "key",
|
||||
"PHOENIX_COLLECTOR_HTTP_ENDPOINT": "https://phoenix.local/v1/traces",
|
||||
},
|
||||
"newrelic": {"NEW_RELIC_LICENSE_KEY": "license"},
|
||||
"agentops": {"AGENTOPS_API_KEY": "ao-key"},
|
||||
"levo": {
|
||||
"LEVOAI_API_KEY": "k",
|
||||
"LEVOAI_ORG_ID": "org",
|
||||
"LEVOAI_WORKSPACE_ID": "ws",
|
||||
"LEVOAI_COLLECTOR_URL": "https://levo.local/v1/traces",
|
||||
},
|
||||
"weave_otel": {"WANDB_API_KEY": "wandb", "WANDB_PROJECT_ID": "entity/project"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestDestinationOwnership:
|
||||
"""A callback exports only to the destination it owns: ``otel`` serves the operator's
|
||||
``OTEL_*`` collector and a preset serves its own backend. The leak this closes was every
|
||||
preset inheriting the collector from the base config, so its vendor-vocabulary spans
|
||||
(``langfuse.*``, OpenInference, ...) landed on the operator's collector too."""
|
||||
|
||||
@staticmethod
|
||||
def _capture_exporters(monkeypatch) -> dict[str, InMemorySpanExporter]:
|
||||
sinks: dict[str, InMemorySpanExporter] = {}
|
||||
|
||||
def factory(spec: ExporterSpec) -> InMemorySpanExporter:
|
||||
return sinks.setdefault(spec.endpoint or "", InMemorySpanExporter())
|
||||
|
||||
swap_exporter_factories(monkeypatch, otlp_http=factory, otlp_grpc=factory, agentops=factory)
|
||||
return sinks
|
||||
|
||||
@staticmethod
|
||||
def _operator_with(monkeypatch, vendor: str) -> None:
|
||||
for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", *_OTEL_SHORTHAND_ENV):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
for name, value in _VENDOR_ENV[vendor].items():
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", _COLLECTOR)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
@pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV))
|
||||
def test_a_presets_spans_never_reach_the_operator_collector(self, monkeypatch, vendor):
|
||||
sinks = self._capture_exporters(monkeypatch)
|
||||
self._operator_with(monkeypatch, vendor)
|
||||
generic = build_otel_v2_logger(OpenTelemetryV2Config())
|
||||
preset = in_fresh_context(_maybe_construct_otel_v2, vendor, [generic])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert preset is not None
|
||||
|
||||
emit(generic.tracer_provider, "from otel")
|
||||
emit(preset.tracer_provider, "from preset")
|
||||
generic.tracer_provider.force_flush()
|
||||
preset.tracer_provider.force_flush()
|
||||
|
||||
assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"]
|
||||
vendor_sinks = {endpoint: sink for endpoint, sink in sinks.items() if endpoint != _COLLECTOR}
|
||||
assert [[s.name for s in sink.get_finished_spans()] for sink in vendor_sinks.values()] == [["from preset"]]
|
||||
|
||||
@pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV))
|
||||
def test_a_preset_built_first_does_not_point_otel_at_its_own_backend(self, monkeypatch, vendor):
|
||||
"""``callbacks: [<vendor>, otel]``: reading the vendor's credentials must not rewrite the
|
||||
process-wide ``OTEL_*`` env, or the ``otel`` built next inherits the vendor's endpoint and auth."""
|
||||
sinks = self._capture_exporters(monkeypatch)
|
||||
self._operator_with(monkeypatch, vendor)
|
||||
preset = in_fresh_context(_maybe_construct_otel_v2, vendor, [])
|
||||
generic = build_otel_v2_logger(OpenTelemetryV2Config())
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert preset is not None
|
||||
|
||||
emit(generic.tracer_provider, "from otel")
|
||||
generic.tracer_provider.force_flush()
|
||||
|
||||
assert [(spec.endpoint, spec.headers) for spec in generic.config.exporters] == [(_COLLECTOR, None)]
|
||||
assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"]
|
||||
|
||||
@pytest.mark.parametrize("otel_first", [True, False])
|
||||
def test_the_otel_entry_never_reuses_a_preset_logger_whatever_the_order(self, monkeypatch, otel_first):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
self._operator_with(monkeypatch, "langfuse_otel")
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)
|
||||
loggers: list[CustomLogger] = []
|
||||
monkeypatch.setattr(litellm_logging, "_in_memory_loggers", loggers)
|
||||
|
||||
def build(name):
|
||||
if name == "otel":
|
||||
return _init_custom_logger_compatible_class("otel", None, None)
|
||||
return _maybe_construct_otel_v2(name, loggers)
|
||||
|
||||
order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel")
|
||||
built = {name: in_fresh_context(build, name) for name in order}
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
generic, preset = built["otel"], built["langfuse_otel"]
|
||||
assert isinstance(generic, OpenTelemetryV2) and isinstance(preset, OpenTelemetryV2)
|
||||
assert generic is not preset
|
||||
assert generic.callback_name is None and preset.callback_name == "langfuse_otel"
|
||||
assert [spec.endpoint for spec in generic.config.exporters] == [_COLLECTOR]
|
||||
assert [spec.endpoint for spec in preset.config.exporters] == ["https://lf.local/api/public/otel"]
|
||||
assert select_global_otel_v2_logger(loggers) is generic
|
||||
assert proxy_server.open_telemetry_logger is generic
|
||||
|
||||
@pytest.mark.parametrize("otel_first", [True, False])
|
||||
def test_the_otel_callback_holds_the_proxy_slot_whatever_the_order(self, monkeypatch, otel_first):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)
|
||||
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")])
|
||||
|
||||
def build(name):
|
||||
return build_otel_v2_logger(config, callback_name=None if name == "otel" else name)
|
||||
|
||||
order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel")
|
||||
built = {name: build(name) for name in order}
|
||||
|
||||
assert proxy_server.open_telemetry_logger is built["otel"]
|
||||
|
||||
def test_a_lone_preset_keeps_the_proxy_slot_and_the_global(self, monkeypatch):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)
|
||||
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")])
|
||||
first = build_otel_v2_logger(config, callback_name="langfuse_otel")
|
||||
second = build_otel_v2_logger(config, callback_name="arize")
|
||||
|
||||
assert proxy_server.open_telemetry_logger is first
|
||||
assert select_global_otel_v2_logger([first, second]) is first
|
||||
|
||||
|
||||
class TestBackendEndpointParity:
|
||||
def test_arize_follows_its_own_http_endpoint_instead_of_the_grpc_default(self, monkeypatch):
|
||||
|
|
@ -2613,75 +2742,6 @@ class TestEvictionSafety:
|
|||
assert held.shutdown_calls == 1
|
||||
|
||||
|
||||
class TestCredentialGatedExporters:
|
||||
def test_layering_a_second_preset_does_not_eat_the_first_gated_exporter(self, monkeypatch):
|
||||
"""``base.Preset`` advertises ``config_overrides`` layering, and the gated spec
|
||||
is itself a console exporter with no endpoint."""
|
||||
credential_less_proxy(monkeypatch)
|
||||
from litellm.integrations.otel.presets.utils import credential_gated_exporters
|
||||
|
||||
once = credential_gated_exporters((), ExporterOwner.LANGFUSE_OTEL)
|
||||
twice = credential_gated_exporters(once, ExporterOwner.WEAVE_OTEL)
|
||||
|
||||
assert [spec.owner for spec in twice] == [ExporterOwner.LANGFUSE_OTEL, ExporterOwner.WEAVE_OTEL]
|
||||
|
||||
def test_an_exporter_the_operator_configured_survives(self):
|
||||
from litellm.integrations.otel.presets.utils import credential_gated_exporters
|
||||
|
||||
operator_console = ExporterSpec(kind="console", use_simple_processor=True)
|
||||
|
||||
kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL)
|
||||
|
||||
assert kept[0] == operator_console
|
||||
|
||||
def test_an_otlp_exporter_on_its_default_endpoint_survives(self):
|
||||
"""``OTEL_EXPORTER=otlp_http`` with no endpoint is a real collector on the SDK's
|
||||
default port, not the placeholder, so the transport is what tells them apart."""
|
||||
from litellm.integrations.otel.presets.utils import credential_gated_exporters
|
||||
|
||||
operator_otlp = ExporterSpec(kind="otlp_http", endpoint=None, headers=None)
|
||||
|
||||
kept = credential_gated_exporters((operator_otlp,), ExporterOwner.LANGFUSE_OTEL)
|
||||
|
||||
assert kept[0] == operator_otlp
|
||||
|
||||
def test_an_in_memory_exporter_the_operator_asked_for_survives(self):
|
||||
"""``OTEL_EXPORTER=in_memory`` stores spans, so it is a destination the operator
|
||||
chose, not the placeholder that stands in for choosing nothing."""
|
||||
from litellm.integrations.otel.presets.utils import credential_gated_exporters
|
||||
|
||||
operator_memory = ExporterSpec(kind="in_memory", endpoint=None, headers=None)
|
||||
|
||||
kept = credential_gated_exporters((operator_memory,), ExporterOwner.LANGFUSE_OTEL)
|
||||
|
||||
assert kept[0] == operator_memory
|
||||
|
||||
def test_the_synthesized_stdout_placeholder_is_dropped(self, monkeypatch):
|
||||
from litellm.integrations.otel.presets.utils import credential_gated_exporters
|
||||
|
||||
for name in _OTEL_SHORTHAND_ENV:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
placeholder = OpenTelemetryV2Config().exporters[0]
|
||||
|
||||
kept = credential_gated_exporters((placeholder,), ExporterOwner.LANGFUSE_OTEL)
|
||||
|
||||
assert [spec.owner for spec in kept] == [ExporterOwner.LANGFUSE_OTEL]
|
||||
|
||||
def test_a_console_exporter_the_operator_named_survives(self, monkeypatch):
|
||||
"""Same kind, endpoint and headers as the placeholder; only the fact that the
|
||||
operator set ``OTEL_EXPORTER`` tells them apart."""
|
||||
from litellm.integrations.otel.presets.utils import credential_gated_exporters
|
||||
|
||||
for name in _OTEL_SHORTHAND_ENV:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setenv("OTEL_EXPORTER", "console")
|
||||
operator_console = OpenTelemetryV2Config().exporters[0]
|
||||
|
||||
kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL)
|
||||
|
||||
assert kept[0] is operator_console
|
||||
|
||||
|
||||
class TestTenantHostSsrfGuard:
|
||||
"""Anyone who can mint a key can write ``langfuse_host``, so the host it names has
|
||||
to be one the operator approved."""
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
"""Multi-backend fan-out: one TracerProvider, *N* SpanProcessors.
|
||||
"""Multi-backend fan-out within one callback: one TracerProvider, *N* SpanProcessors.
|
||||
|
||||
V1 needed a separate ``TracerProvider`` per integration to avoid stepping on
|
||||
the global. V2 attaches a ``SpanProcessor`` per exporter to the *same*
|
||||
provider, so the same trace ID lights up every backend — no duplicate spans,
|
||||
no per-integration provider caches.
|
||||
A single ``OpenTelemetryV2Config`` may list several exporters (say two ``OTEL_*``
|
||||
collectors), and its provider attaches one ``SpanProcessor`` per exporter so the same
|
||||
span reaches each of them. Which exporters a config lists is decided per callback: a
|
||||
preset only lists the destination it owns, see ``TestDestinationOwnership``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
|
@ -14,8 +14,8 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
|||
InMemorySpanExporter,
|
||||
)
|
||||
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.plumbing.providers import build_tracer_provider, register_exporter_factory
|
||||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.plumbing.providers import build_tracer_provider
|
||||
|
||||
|
||||
def test_two_exporters_receive_the_same_span():
|
||||
|
|
@ -54,52 +54,6 @@ def test_two_exporters_receive_the_same_span():
|
|||
assert spans_a[0].context.span_id == spans_b[0].context.span_id
|
||||
|
||||
|
||||
def _capturing_kind(kind):
|
||||
exporter = InMemorySpanExporter()
|
||||
register_exporter_factory(kind, lambda _spec: exporter)
|
||||
return exporter
|
||||
|
||||
|
||||
def _emit_langfuse_stamped_span(provider):
|
||||
span = provider.get_tracer("test").start_span("chat proof-model")
|
||||
span.set_attribute("gen_ai.request.model", "proof-model")
|
||||
span.set_attribute("langfuse.trace.name", "private-langfuse-only-name")
|
||||
span.set_attribute("langfuse.observation.input", '[{"role":"user","content":"hi"}]')
|
||||
span.end()
|
||||
return span.context.span_id
|
||||
|
||||
|
||||
def test_generic_exporter_does_not_receive_langfuse_attributes_the_langfuse_exporter_keeps():
|
||||
generic = _capturing_kind("capture_generic_collector")
|
||||
langfuse = _capturing_kind("capture_langfuse_sink")
|
||||
cfg = OpenTelemetryV2Config(
|
||||
exporters=[
|
||||
ExporterSpec(kind="capture_generic_collector"),
|
||||
ExporterSpec(kind="capture_langfuse_sink", owner=ExporterOwner.LANGFUSE_OTEL),
|
||||
]
|
||||
)
|
||||
span_id = _emit_langfuse_stamped_span(build_tracer_provider(cfg, tenant_overrides=True))
|
||||
|
||||
(generic_span,) = generic.get_finished_spans()
|
||||
(langfuse_span,) = langfuse.get_finished_spans()
|
||||
assert generic_span.context.span_id == span_id == langfuse_span.context.span_id
|
||||
assert generic_span.attributes["gen_ai.request.model"] == "proof-model"
|
||||
assert [k for k in generic_span.attributes if k.startswith("langfuse.")] == []
|
||||
assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
|
||||
assert langfuse_span.attributes["langfuse.observation.input"] == '[{"role":"user","content":"hi"}]'
|
||||
|
||||
|
||||
def test_langfuse_mapper_on_a_plain_otlp_exporter_keeps_langfuse_attributes():
|
||||
sink = _capturing_kind("capture_langfuse_mapper_sink")
|
||||
cfg = OpenTelemetryV2Config(
|
||||
mapper_names=["langfuse"], exporters=[ExporterSpec(kind="capture_langfuse_mapper_sink")]
|
||||
)
|
||||
_emit_langfuse_stamped_span(build_tracer_provider(cfg))
|
||||
|
||||
(span,) = sink.get_finished_spans()
|
||||
assert span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
|
||||
|
||||
|
||||
def test_resource_attributes_apply_to_all_exporters():
|
||||
"""``resource_attributes`` flow through the shared TracerProvider."""
|
||||
cfg = OpenTelemetryV2Config(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue