diff --git a/litellm/__init__.py b/litellm/__init__.py index 15e95ded906..ae5f278076d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -136,6 +136,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "langfuse", "langfuse_otel", "weave_otel", + "generic", "pagerduty", "humanloop", "azure_sentinel", diff --git a/litellm/constants.py b/litellm/constants.py index aeb74a65839..e61078ce636 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -424,6 +424,11 @@ DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE = os.getenv( LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED = 499 +# Reserved key/team logging callback var that binds the callback to a named OTEL +# credential in the registry (an admin-owned reference resolved server-side into a +# trace destination, never forwarded as a request parameter). +LITELLM_LOGGING_CREDENTIAL_NAME_KEY = "litellm_logging_credential_name" + EMAIL_BUDGET_ALERT_TTL = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)) # 24 hours in seconds EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float( os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index de543fa042b..c7e5e83a6f2 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -3029,6 +3029,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if signal_type == "traces" and "/v2/trace/otlp" in endpoint: return endpoint + # Langtrace ingests traces at /api/trace (a complete path, not an OTLP base). Do not rewrite. + if signal_type == "traces" and endpoint.endswith("/api/trace"): + return endpoint + # Check if endpoint already ends with the correct signal path target_path = f"/v1/{signal_type}" if endpoint.endswith(target_path): diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 17011bb8db7..fb584ccdb15 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -247,12 +247,13 @@ lives in [`plumbing/`](./plumbing): - [`presets/`](./presets) — each preset reads one integration's env vars and returns an `OpenTelemetryV2Config` (exporter destination + mapper vocabularies + resource attributes). `PRESET_BY_CALLBACK` maps a callback name (`"arize"`, - `"langfuse_otel"`, …) to its preset. Integrations that support team/key-scoped - credentials also provide a per-request OTLP header builder - (`DYNAMIC_HEADERS_BY_CALLBACK`). Presets do **no** network I/O at build time: - AgentOps, for example, mints its JWT lazily inside a custom exporter on the - first export (in the `BatchSpanProcessor` worker thread), never on the event - loop. + `"langfuse_otel"`, …) to its preset. Per-key/team routing is **not** here: a + destination is admin-owned infrastructure, resolved server-side from a named + credential into an `OtelDestination` (`destinations.py`) and applied by + `plumbing/routing.py`. Nothing in `presets/` reads vendor credentials or a host + off a request. Presets do **no** network I/O at build time: AgentOps, for + example, mints its JWT lazily inside a custom exporter on the first export (in + the `BatchSpanProcessor` worker thread), never on the event loop. ## Extending @@ -261,7 +262,8 @@ lives in [`plumbing/`](./plumbing): `key -> extractor` tables) and register it in `mappers/__init__._MAPPER_BY_NAME`. - **A new integration**: add a preset in `presets/` that returns an `OpenTelemetryV2Config`, and register it in `presets/__init__.PRESET_BY_CALLBACK`. - If it supports dynamic credentials, add a header builder to - `DYNAMIC_HEADERS_BY_CALLBACK`. + For admin-owned per-key/team destinations, add an adapter mapping the named + credential's values to an `OtelDestination` in `destinations._ADAPTERS` (or rely + on the generic `otel_endpoint`/`otel_headers` passthrough). - **A new span kind**: add a role to `spans.py` (registry entry + name builder), a payload dataclass in `payloads.py`, and a branch in the relevant mapper(s). diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 69fc53c5b9d..8ee1aaa8710 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -138,6 +138,43 @@ class SpanEmitter: self.finish_span(role, span, data, end_time_ns=end_time_ns) return span + def emit_fanout( + self, + role: SpanRole, + data: SpanData, + parent_context: Context | None = None, + *, + start_time_ns: int | None = None, + end_time_ns: int | None = None, + tracers: Sequence[Tracer], + ) -> Span | None: + """Emit one logical span once per tracer, deduping the call ONCE. + + A backend that selects its target from the Resource (Arize's project) needs a + separately-tagged span per Resource group, so ``tracers`` is one tracer per group + (from ``TenantTracerCache.tracers_for``). Dedup runs once on the call id so a + sync+async double-firing still coalesces, then a span is started and finished on + each tracer (each provider stamps its own Resource). Returns the first span (or + ``None`` if deduped/empty). + """ + dedup_key = data.identity.call_id if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData)) else None + if self._seen(dedup_key, role): + return None + name = _NAME_BUILDERS[role](data) + first: Span | None = None + for tracer in tracers: + span = self.start_span( + role, + name, + parent_context=parent_context, + start_time_ns=start_time_ns, + tracer=tracer, + ) + self.finish_span(role, span, data, end_time_ns=end_time_ns) + if first is None: + first = span + return first + def finish_span( self, role: SpanRole, diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 44484559948..8d5bbea41f5 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -52,6 +52,7 @@ from litellm.integrations.otel.model.spans import SpanRole, span_role_for_servic from litellm.integrations.otel.model.utils import to_ns if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination from litellm.types.utils import ( StandardLoggingGuardrailInformation, StandardLoggingPayload, @@ -78,18 +79,19 @@ _OPEN_CALLS_MAX = 10_000 class _LLMCallSpan: """The state carried from the ``pre_call`` boundary to span close. - ``span`` is the live span when it could be opened at the boundary (the server - span was ambient), or ``None`` when creation was deferred because no ambient - parent was visible — in which case the async callback creates it against its - own (worker-copied) ambient context using ``start_time_ns``. The presence of - a carrier for a call at all is the proof that ``pre_call`` ran, i.e. that an - upstream call was actually attempted. + ``spans`` is one live span per destination Resource group (a backend like Arize + routing two projects yields two), opened at the boundary when the server span was + ambient. It is empty when creation was deferred because no ambient parent was visible + — in which case the async callback creates the span(s) against its own + (worker-copied) ambient context using ``start_time_ns``. The presence of a carrier + for a call at all is the proof that ``pre_call`` ran, i.e. that an upstream call was + actually attempted. """ - __slots__ = ("span", "start_time_ns") + __slots__ = ("spans", "start_time_ns") - def __init__(self, span: "Span | None", start_time_ns: int | None) -> None: - self.span = span + def __init__(self, spans: "tuple[Span, ...]", start_time_ns: int | None) -> None: + self.spans = spans self.start_time_ns = start_time_ns @@ -109,7 +111,13 @@ class OpenTelemetryV2(CustomLogger): self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs) self.callback_name = callback_name self._tracer_provider: TracerProvider = ( - tracer_provider if tracer_provider is not None else build_tracer_provider(self.config) + tracer_provider + if tracer_provider is not None + else build_tracer_provider( + self.config, + tenant_fan_out_owner=callback_name, + attach_tenant_fan_out=True, + ) ) self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) @@ -117,6 +125,11 @@ class OpenTelemetryV2(CustomLogger): self._emitter = SpanEmitter(self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)) self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME) self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict() + # call_ids for which the LLM-call span has already been emitted; lets + # _close_llm_call no-op on duplicate callbacks (success + failure both + # firing, or success firing twice) instead of double-exporting the + # deferred-emit span. Bounded LRU, same size as _open_llm_calls. + self._closed_call_ids: "OrderedDict[str, None]" = OrderedDict() self._init_otel_logger_on_litellm_proxy() def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None": @@ -159,6 +172,15 @@ class OpenTelemetryV2(CustomLogger): if getattr(proxy_server, "open_telemetry_logger", None) is None: setattr(proxy_server, "open_telemetry_logger", self) + def _destinations_for_backend(self, call: "LLMCallEvent") -> tuple: + """The call's admin-resolved destinations that belong to THIS logger's backend. + + A request fans out across whatever exporters its identity chain is assigned; + each logger exports only the destinations tagged with its own callback_name, + so each backend's span keeps its own attribute vocabulary. + """ + return tuple(d for d in call.otel_destinations if d.callback_name == self.callback_name) + # ====================================================================== # # LLM-call callbacks — the span is opened at the ``pre_call`` boundary and # closed here. See ``log_pre_api_call``. @@ -194,21 +216,27 @@ class OpenTelemetryV2(CustomLogger): if call_id in self._open_llm_calls: return start_time_ns = to_ns(datetime.now()) - span: Span | None = None + # One live span per destination Resource group (Arize routing two projects + # yields two; header-routed backends yield one). Empty until a recordable + # parent is confirmed. + spans: tuple[Span, ...] = () # Parent to the request's anchored root span (stable across the request), # falling back to ambient on the SDK path. Open the span live only when # that resolves to a recordable parent; otherwise defer to the close # callback (the thread-pool case, where the anchor isn't visible here). parent_context = resolve_request_span_context() if is_recordable_span(get_current_span(parent_context)): - span = self._emitter.start_span( - SpanRole.LLM_CALL, - call.provisional_span_name, - parent_context=parent_context, - start_time_ns=start_time_ns, - tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params), + spans = tuple( + self._emitter.start_span( + SpanRole.LLM_CALL, + call.provisional_span_name, + parent_context=parent_context, + start_time_ns=start_time_ns, + tracer=tracer, + ) + for tracer in self._tenant_tracers.tracers_for(self.tracer, self._destinations_for_backend(call)) ) - self._open_llm_calls[call_id] = _LLMCallSpan(span=span, start_time_ns=start_time_ns) + self._open_llm_calls[call_id] = _LLMCallSpan(spans=spans, start_time_ns=start_time_ns) # Evict the oldest open call if the map is over budget. A call that opens # but never closes (a stream that only fires stream events) would linger # otherwise; the evicted span is simply dropped (never exported). @@ -288,38 +316,85 @@ class OpenTelemetryV2(CustomLogger): ) -> Span | None: """Finish the LLM-call span opened at ``pre_call`` (or create it deferred). - No carrier for this call id means ``pre_call`` never ran — the request was - rejected at the gate or blocked by a pre-call guardrail before any upstream - call — so there is nothing to record and no phantom span. + Missing carrier has two shapes. ``pre_call`` genuinely never ran -- the + request was rejected at the gate or blocked by a pre-call guardrail before + any upstream call, so no payload exists and dropping is correct. OR this + v2 instance was lazily activated AFTER ``pre_call`` iterated the callback + list (the destination-resolver path: a credential resolved a backend the + YAML didn't pre-list), so the upstream call DID happen, the payload IS + set, and the admin-resolved destinations name this backend -- emit a + deferred span with the success event's start time so the per-tenant + exporter ships it. """ call = LLMCallEvent.from_dict(kwargs) call_id = call.call_id - # ``pop`` is the dedup: this method runs from both the success and failure - # paths, and whichever fires first removes the carrier and closes the span. + + # Dedup guard: a normal close pops the carrier and emits a span. If a + # second close fires for the same call_id (success + failure callbacks + # are wired separately, custom callbacks can fan out, etc.), the + # carrier is already gone and a payload+destinations combination would + # otherwise emit a second deferred span. + if call_id and call_id in self._closed_call_ids: + return None + carrier = self._open_llm_calls.pop(call_id, None) if call_id else None - if carrier is None: - return None payload = call.payload - if payload is None: - if carrier.span is not None: - # Opened at the boundary but the payload never materialized — end - # it (named provisionally) so it isn't leaked as an open span. - carrier.span.end(end_time=to_ns(end_time)) - return None - data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=self.config.capture_span_content) + + if carrier is None: + destinations = self._destinations_for_backend(call) + if payload is None or not destinations: + return None + self._mark_closed(call_id) + return self._emit_deferred_llm_call(payload, destinations, to_ns(start_time), to_ns(end_time)) + end_time_ns = to_ns(end_time) - if carrier.span is not None: - # Born at the boundary: stamp attributes from the typed payload, set - # status, and end it. Its parent (the server span) was captured at - # creation from real ambient context. - self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns) - return carrier.span - # Deferred: ``pre_call`` saw no recordable parent, so create the span now. - # The worker copied the request task's context, which carries the anchored - # root span — parent to it (ambient fallback on the SDK path). Seed identity - # Baggage so the span — and the SDK path, which has none — is labeled - # consistently. - parent_ctx = resolve_request_span_context() + self._mark_closed(call_id) + if payload is None: + for span in carrier.spans: + span.end(end_time=end_time_ns) + return None + + data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=self.config.capture_span_content) + if carrier.spans: + for span in carrier.spans: + self._emitter.finish_span(SpanRole.LLM_CALL, span, data, end_time_ns=end_time_ns) + return carrier.spans[0] + return self._emit_deferred_llm_call( + payload, + self._destinations_for_backend(call), + carrier.start_time_ns, + end_time_ns, + ) + + def _mark_closed(self, call_id: str | None) -> None: + """Remember a call_id has been closed so a duplicate callback no-ops. + + Bounded by the same ceiling as ``_open_llm_calls`` to prevent unbounded + growth; oldest entries are evicted FIFO. + """ + if not call_id: + return + self._closed_call_ids[call_id] = None + if len(self._closed_call_ids) > _OPEN_CALLS_MAX: + self._closed_call_ids.popitem(last=False) + + def _emit_deferred_llm_call( + self, + payload: "StandardLoggingPayload", + destinations: "tuple[OtelDestination, ...]", + start_time_ns: int | None, + end_time_ns: int | None, + ) -> Span | None: + """Emit an LLM-call span outside the ``pre_call`` boundary. + + Two callers: the SDK thread-pool path (carrier existed but ``pre_call`` + saw no recordable parent) and the destination-resolver path (this v2 + instance was born after ``pre_call`` ran, so no carrier was ever opened). + Both anchor to the request's root span via the worker-copied context and + seed identity Baggage so the span is labeled consistently. + """ + data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=self.config.capture_span_content) + base_ctx = resolve_request_span_context() bag = promoted_baggage( data.identity, data.request_model, @@ -327,15 +402,14 @@ class OpenTelemetryV2(CustomLogger): metadata_keys=tuple(self.config.baggage_metadata_keys), team_metadata_keys=tuple(self.config.baggage_team_metadata_keys), ) - if bag: - parent_ctx = set_request_baggage(bag, context=parent_ctx) - return self._emitter.emit( + parent_ctx = set_request_baggage(bag, context=base_ctx) if bag else base_ctx + return self._emitter.emit_fanout( SpanRole.LLM_CALL, data, parent_context=parent_ctx, - start_time_ns=carrier.start_time_ns, + start_time_ns=start_time_ns, end_time_ns=end_time_ns, - tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params), + tracers=self._tenant_tracers.tracers_for(self.tracer, destinations), ) # ====================================================================== # diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py new file mode 100644 index 00000000000..3341c2cec02 --- /dev/null +++ b/litellm/integrations/otel/model/destination.py @@ -0,0 +1,27 @@ +"""The resolved, admin-owned OTLP destination. + +A trace destination is admin-owned infrastructure config, never request data. +The proxy resolves a key/team's bound named credential into this typed, +backend-agnostic target (an endpoint plus auth headers) server-side, and the v2 +logger exports through it. Every OTEL backend -- Langfuse, Arize, Weave, a +self-hosted collector -- reduces to this shape; the per-backend field mapping +lives in ``litellm.integrations.otel.destinations``. +""" + +from pydantic import BaseModel, ConfigDict, Field + + +class OtelDestination(BaseModel): + model_config = ConfigDict(frozen=True) + + endpoint: str + headers: dict[str, str] = Field(default_factory=dict) + resource_attributes: dict[str, str] = Field(default_factory=dict) + # The OTEL backend (callback_name) this destination belongs to, so a request + # that fans out across backends routes each destination to the logger that + # owns its attribute vocabulary. None for the legacy single-destination path. + callback_name: str | None = None + + def header_string(self) -> str: + """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects.""" + return ",".join(f"{key}={value}" for key, value in self.headers.items()) diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 37bb5464315..5db88d25bc4 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -39,7 +39,10 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Mapping, cast +from pydantic import ValidationError + from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL +from litellm.integrations.otel.model.destination import OtelDestination from litellm.integrations.otel.model.semconv import resolve_operation from litellm.integrations.otel.model.utils import as_str @@ -47,6 +50,31 @@ if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload +def _otel_destinations(dynamic_params: Any) -> tuple[OtelDestination, ...]: + """The admin-resolved OTLP destinations carried on ``standard_callback_dynamic_params``. + + Server-set only (the proxy resolves the exporters assigned to the request's + identity chain and strips any client value), so this is the sole source the v2 + router trusts -- request-supplied vendor credentials are never read here. A + request fans out to every destination here; each logger keeps only the ones + tagged with its own backend. + """ + if not isinstance(dynamic_params, Mapping): + return () + raw = dynamic_params.get("otel_destinations") + if not isinstance(raw, list): + return () + parsed: list[OtelDestination] = [] + for item in raw: + if not isinstance(item, Mapping): + continue + try: + parsed.append(OtelDestination.model_validate(dict(item))) + except ValidationError: + continue + return tuple(parsed) + + @dataclass(frozen=True) class RequestIdentity: call_id: str | None = None @@ -194,6 +222,10 @@ class LLMCallEvent: # The ``standard_callback_dynamic_params`` routing the call to a per-tenant # tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped. dynamic_params: Any + # The admin-resolved OTLP destinations (endpoint + auth headers) for this call's + # identity chain, fanned out to. Empty when none are assigned. The only source the + # v2 router trusts for per-tenant routing; never request-derived. + otel_destinations: tuple[OtelDestination, ...] # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire # the ``pre_call`` hook but never made an upstream call, so they get no span. is_no_upstream_call: bool @@ -208,10 +240,12 @@ class LLMCallEvent: payload = cast("StandardLoggingPayload", raw_payload) if raw_payload else None operation = resolve_operation(as_str(kwargs.get("call_type"))) model = as_str(kwargs.get("model")) or "" + dynamic_params = kwargs.get("standard_callback_dynamic_params") return cls( call_id=_call_id(payload, kwargs), payload=payload, - dynamic_params=kwargs.get("standard_callback_dynamic_params"), + dynamic_params=dynamic_params, + otel_destinations=_otel_destinations(dynamic_params), is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)), provisional_span_name=f"{operation.value} {model}".strip(), ) diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index ebcf4aa35af..074fc928e5e 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -101,17 +101,27 @@ def instrument_fastapi_app(app: Any) -> None: after config load (see ``proxy_startup_event``), and the proxy delegates to it. That way server spans and gen-ai spans share one provider and the same trace. """ + if not is_otel_v2_enabled(): + return + + # Lazy: only the V2-enabled path needs the optional + # ``opentelemetry-instrumentation-fastapi`` package. Importing it at module top + # would make ``proxy_server``'s unconditional ``import`` of this module crash when + # the package is absent, even with the gate off. When V2 IS on, a missing package + # is a real misconfiguration -- without the server span the trace has no root and + # admin-owned destination traces are orphaned -- so it must be loud, not silent. try: - if not is_otel_v2_enabled(): - return - - # Lazy: only the V2-enabled path needs the optional - # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the - # base ``litellm[proxy]`` install. Importing it at module top would make - # ``proxy_server``'s unconditional ``import`` of this module crash when the - # package is absent, even with the gate off. from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + except ImportError: + verbose_logger.warning( + "LITELLM_OTEL_V2 is enabled but 'opentelemetry-instrumentation-fastapi' " + "is not installed. The FastAPI server span will not be created, so traces " + "exported to admin-owned destinations will be missing their root span " + "(orphaned children). Install 'opentelemetry-instrumentation-fastapi'." + ) + return + try: excluded_urls = ( os.environ.get("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS") if "OTEL_PYTHON_FASTAPI_EXCLUDED_URLS" in os.environ @@ -127,4 +137,4 @@ def instrument_fastapi_app(app: Any) -> None: exclude_spans=["receive", "send"], ) except Exception as e: - verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: %s", e) + verbose_logger.warning("OTel V2 FastAPI instrumentation failed: %s", e) diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index ff513c84d95..1059c939250 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -1,7 +1,7 @@ """Trace-context + Baggage helpers.""" from contextvars import ContextVar -from typing import Mapping +from typing import TYPE_CHECKING, Mapping from opentelemetry import baggage from opentelemetry.context import Context, get_current @@ -10,6 +10,9 @@ from opentelemetry.trace.propagation.tracecontext import ( TraceContextTextMapPropagator, ) +if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination + _PROPAGATOR = TraceContextTextMapPropagator() # The request's root span — the FastAPI-owned SERVER span — captured ONCE when the @@ -29,6 +32,31 @@ _PROPAGATOR = TraceContextTextMapPropagator() # request task, so there is nothing to leak. _request_root_span: "ContextVar[Span | None]" = ContextVar("litellm_otel_request_root_span", default=None) +# Per-request admin-resolved destinations. Set once at the auth boundary (the +# earliest point a request's identity is known) and read by the global-provider +# fan-out processor at ``on_end`` time, so every span the proxy emits for this +# request -- the FastAPI server span, the ``auth`` phase, DB lookups, the +# batch-write cost ledger -- ships to every per-tenant destination the admin +# assigned. Lives on a ``ContextVar`` so it follows the request task across +# ``asyncio.create_task`` children (the success/failure logging callbacks close +# the LLM span in a worker copied from the request context). Request-scoped: the +# contextvar dies with the request task, so nothing leaks across requests. +_request_destinations: 'ContextVar[tuple["OtelDestination", ...]]' = ContextVar( + "litellm_otel_request_destinations", default=() +) + + +def set_request_destinations( + destinations: 'tuple["OtelDestination", ...]', +) -> None: + """Anchor the admin-resolved destinations for this request.""" + _request_destinations.set(tuple(destinations)) + + +def request_destinations() -> 'tuple["OtelDestination", ...]': + """Destinations the request fans out to, or empty when none were resolved.""" + return _request_destinations.get() + def set_request_root_span(span: Span) -> None: """Anchor the request's root (server) span for explicit child parenting. diff --git a/litellm/integrations/otel/plumbing/fan_out.py b/litellm/integrations/otel/plumbing/fan_out.py new file mode 100644 index 00000000000..5a4b35a67fa --- /dev/null +++ b/litellm/integrations/otel/plumbing/fan_out.py @@ -0,0 +1,204 @@ +"""Per-request span fan-out to admin-resolved destinations. + +Attached to the main ``TracerProvider`` so every span on it (the FastAPI server +span, the proxy's ``auth`` phase span, DB lookups, the post-call cost ledger) +ships to every per-tenant destination the admin assigned for this request, on +top of the provider's configured global exporters. + +Spans emitted through the per-tenant ``TenantTracerCache`` clone providers (the +gen-AI LLM-call span and its MCP-tool sibling) reach tenant backends through +the clone's own exporters; the clone's provider has its own processor list and +does NOT carry this fan-out processor, so a span is exported once per backend. + +Each backend often requires backend-specific Resource attributes (Arize rejects +spans missing ``model_id`` / ``arize.project.name``), so the fan-out wraps each +forwarded span with the destination's expected Resource before handing it to +the per-destination exporter. +""" + +from __future__ import annotations + +from collections import OrderedDict +from typing import TYPE_CHECKING + +from opentelemetry.context import Context +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor + +from litellm._logging import verbose_logger +from litellm.integrations.otel.model.config import ExporterSpec +from litellm.integrations.otel.plumbing.context import request_destinations + +if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination + +# Bound on cached per-destination processors. One processor per +# ``(endpoint, sorted(headers))`` pair, so the working set is one entry per +# admin-resolved tenant credential -- a real-world deployment with hundreds of +# tenants stays well under this. Evicted entries are dropped (not shut down; see +# the eviction site) and reclaimed at process exit. +_MAX_CACHED_PROCESSORS = 256 + + +def _processor_key(destination: OtelDestination) -> tuple: + return (destination.endpoint, tuple(sorted(destination.headers.items()))) + + +class TenantFanOutSpanProcessor(SpanProcessor): + """Forward each finished span to every admin-resolved per-tenant destination. + + The destinations are looked up from a request-scoped contextvar set during + auth, so the processor is stateless across requests and isolation across + concurrent requests is guaranteed by Python's contextvars. + """ + + def __init__(self, owner_callback_name: str | None) -> None: + self._owner = owner_callback_name + # Built lazily so we avoid importing the providers module at class + # definition time (which would create a circular import with routing). + self._processors: OrderedDict[tuple, SpanProcessor] = OrderedDict() + + def on_start(self, span: Span, parent_context: Context | None = None) -> None: + return None + + def on_end(self, span: ReadableSpan) -> None: + destinations = request_destinations() + if not destinations: + return + # The gen-AI LLM-call span (and the MCP tool-call sibling) is already + # routed to per-tenant destinations by the per-backend v2 logger via + # ``TenantTracerCache`` -- the logger picks the right attribute mapper + # (OpenInference for arize, GenAI semconv for langfuse_otel) and ships + # through the clone provider's appended exporter. Forwarding it here too + # would deliver a SECOND copy with the wrong vocabulary and a fresh + # span_id, surfacing in the destination as an orphaned duplicate. Skip. + if _is_genai_span(span): + return + # Proxy-internal spans (FastAPI server, ``auth`` phase, postgres lookups, + # post-call cost ledger) are generic OTel semantic-convention spans with + # no backend-specific vocabulary, so they ship to EVERY admin-resolved + # destination this request fans out to, regardless of the destination's + # ``callback_name``. The owner discriminator only matters for the gen-AI + # span (handled by the skip above). + for destination in destinations: + processor = self._processor_for(destination) + if processor is None: + continue + try: + processor.on_end(_with_destination_resource(span, destination)) + except Exception as exc: # noqa: BLE001 + verbose_logger.debug( + "OTel V2 fan-out: forwarding span to %s failed: %s", + destination.endpoint, + exc, + ) + + def shutdown(self) -> None: + for processor in self._processors.values(): + try: + processor.shutdown() + except Exception as exc: # noqa: BLE001 + verbose_logger.debug("OTel V2 fan-out: processor shutdown failed: %s", exc) + self._processors.clear() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + all_ok = True + for processor in self._processors.values(): + try: + if not processor.force_flush(timeout_millis): + all_ok = False + except Exception: # noqa: BLE001 + all_ok = False + return all_ok + + def _processor_for(self, destination: OtelDestination) -> SpanProcessor | None: + key = _processor_key(destination) + cached = self._processors.get(key) + if cached is not None: + self._processors.move_to_end(key) + return cached + from litellm.integrations.otel.plumbing.providers import ( + _exporter_from_spec, + _processor_for as _build_processor, + default_otlp_kind_for_backend, + ) + + try: + spec = ExporterSpec( + kind=default_otlp_kind_for_backend(destination.callback_name), + endpoint=destination.endpoint, + headers=destination.header_string(), + owner=None, + ) + exporter = _exporter_from_spec(spec) + processor = _build_processor(exporter, use_simple=False) + except Exception as exc: + verbose_logger.debug( + "OTel V2 fan-out: failed to build processor for %s: %s", + destination.endpoint, + exc, + ) + return None + self._processors[key] = processor + if len(self._processors) > _MAX_CACHED_PROCESSORS: + # Evict the LRU entry but do NOT shut it down here: a + # ``BatchSpanProcessor`` may still hold spans queued on its exporter + # thread, and calling ``shutdown`` synchronously can drop or raise on + # those in-flight spans. Dropping the reference lets the worker drain + # naturally and be reclaimed at process exit. The cache is bounded at + # ``_MAX_CACHED_PROCESSORS``, so the un-shut-down working set stays + # bounded. + self._processors.popitem(last=False) + return processor + + +# Attribute set on every gen-AI LLM-call span by the v2 emitter. Used as the +# unambiguous skip signal: only the LLM-call span carries this, and the +# per-backend v2 logger already routes it to per-tenant destinations through +# the TenantTracerCache clone provider's appended exporter. +_GENAI_SPAN_ATTR = "gen_ai.operation.name" + + +def _is_genai_span(span: ReadableSpan) -> bool: + attributes = span.attributes or {} + return _GENAI_SPAN_ATTR in attributes + + +def _with_destination_resource(span: ReadableSpan, destination: OtelDestination) -> ReadableSpan: + """Return ``span`` with its Resource augmented by the destination's required + attributes. The original span object is left untouched; a shallow wrapper + reuses every other field and only swaps the ``resource`` property.""" + from litellm.integrations.otel.plumbing.providers import ( + destination_resource_attrs, + ) + + extra = destination_resource_attrs(destination) + if not extra: + return span + merged = Resource.create({**dict(span.resource.attributes), **extra}) + return _ResourceWrappedReadableSpan(span, merged) + + +class _ResourceWrappedReadableSpan(ReadableSpan): + """A ``ReadableSpan`` view whose ``resource`` is overridden. + + The OTLP exporter reads each span's ``resource`` when serializing; for + backend-specific attributes (Arize's ``model_id``) we substitute the + destination's expected Resource without mutating the underlying span. + """ + + def __init__(self, inner: ReadableSpan, resource: Resource) -> None: + super().__init__( + name=inner.name, + context=inner.context, + parent=inner.parent, + resource=resource, + attributes=inner.attributes, + events=inner.events, + links=inner.links, + kind=inner.kind, + status=inner.status, + start_time=inner.start_time, + end_time=inner.end_time, + instrumentation_scope=inner.instrumentation_scope, + ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ac971c6daa8..90fb28a95ab 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -31,6 +31,8 @@ if TYPE_CHECKING: from opentelemetry.metrics import Meter from opentelemetry.sdk.metrics.export import MetricReader + from litellm.integrations.otel.model.destination import OtelDestination + _SPAN_KIND_BY_ROLE_KIND: dict[LiteLLMSpanKind, SpanKind] = { LiteLLMSpanKind.SERVER: SpanKind.SERVER, LiteLLMSpanKind.CLIENT: SpanKind.CLIENT, @@ -99,8 +101,11 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None: if not endpoint: return endpoint endpoint = endpoint.rstrip("/") - # Splunk Observability uses ``/v2/trace/otlp``; never rewrite it. - if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint: + # Some vendors expose a complete traces ingest path that is NOT the OTLP-standard + # ``/v1/traces`` base: Splunk Observability uses ``/v2/trace/otlp`` and Langtrace + # ingests at ``/api/trace``. Appending ``/v1/traces`` to those 404s, so never + # rewrite them. + if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint or endpoint.endswith("/api/trace"): return endpoint for other_signal in ("/v1/logs", "/v1/metrics"): if endpoint.endswith(other_signal): @@ -108,6 +113,37 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None: return endpoint + "/v1/traces" +# Backends whose OTLP transport is gRPC. Arize's OTLP endpoint +# (``otlp.arize.com``) speaks gRPC; every other current preset speaks OTLP/HTTP. +# Single source of truth shared by the per-tenant fan-out processor and the +# ``TenantTracerCache`` so the two never disagree on a destination's transport. +_GRPC_BACKENDS = frozenset({"arize"}) + + +def default_otlp_kind_for_backend(callback_name: "str | None") -> str: + """The intrinsic OTLP transport for a backend's own OTLP endpoint.""" + return "otlp_grpc" if callback_name in _GRPC_BACKENDS else "otlp_http" + + +def destination_resource_attrs(destination: "OtelDestination") -> dict[str, str]: + """The backend-required Resource attributes a destination carries on every span. + + Backend-agnostic: each backend's destination builder (``presets.destinations``) + declares whatever Resource attributes its ingestion needs, and this just reads + them. Backends that route by auth header (langfuse, weave, generic OTLP) declare + none; Arize declares ``model_id`` / ``arize.project.name`` because it selects the + project from the Resource, not a header. New backends needing Resource-level + routing only have to populate ``resource_attributes`` in their builder. + + Shared by the two export paths that reach a per-tenant destination -- the + ``TenantFanOutSpanProcessor`` (proxy-internal spans) and the ``TenantTracerCache`` + clone provider (the gen-AI span) -- so the gen-AI span and its parents always + carry the SAME Resource and a backend like Arize renders one connected trace + instead of an orphaned subtree. + """ + return dict(destination.resource_attributes) + + def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter: kind = (spec.kind or "console").lower() factory = _EXPORTER_FACTORIES.get(kind) @@ -282,6 +318,8 @@ def build_tracer_provider( exporter: SpanExporter | None = None, baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, + tenant_fan_out_owner: str | None = None, + attach_tenant_fan_out: bool = False, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -290,12 +328,30 @@ def build_tracer_provider( ``config.exporters`` entry — this is what fans spans out to multiple backends. ``exporter`` and ``use_simple_processor`` are explicit overrides: pass a single exporter to attach exactly that one (used by tests). + + ``attach_tenant_fan_out`` — attach a ``TenantFanOutSpanProcessor`` that forwards + each finished proxy-internal span (FastAPI server, ``auth`` phase, DB lookups, the + cost ledger) to the request's admin-resolved destinations. The MAIN v2 logger + provider always opts in, EVEN when no backend is named (the generic global logger + published for a destination-only deployment) -- otherwise the server span never + reaches the destination and its gen-AI child is orphaned. ``tenant_fan_out_owner`` + is the owning backend name when one exists; it is informational (the fan-out skips + the gen-AI span by attribute and forwards internal spans to every destination). + The per-tenant clone providers (built by ``TenantTracerCache``) pass neither, so + the LLM-call span exported through them is not also fanned out here. """ provider = TracerProvider(resource=build_resource(config)) if baggage_processor is None: baggage_processor = LiteLLMBaggageSpanProcessor(allowed_keys=config.baggage_promoted_keys) provider.add_span_processor(baggage_processor) + if attach_tenant_fan_out or tenant_fan_out_owner is not None: + from litellm.integrations.otel.plumbing.fan_out import ( + TenantFanOutSpanProcessor, + ) + + provider.add_span_processor(TenantFanOutSpanProcessor(owner_callback_name=tenant_fan_out_owner)) + if exporter is not None: provider.add_span_processor(_processor_for(exporter, use_simple_processor)) return provider diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 2f8945e903b..37104331b98 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -1,36 +1,41 @@ -"""Per-request multi-tenant tracer routing. +"""Per-request multi-tenant tracer routing with fan-out. -When a request carries team/key vendor credentials in -``standard_callback_dynamic_params``, its spans must export through a -``TracerProvider`` whose OTLP headers carry those credentials. -``TenantTracerCache`` builds and caches one provider per distinct credential -set, and otherwise hands back the logger's default tracer. This lets a single -logger fan requests out to many tenants without needing a logger per tenant. +A call's identity chain is assigned a set of admin-owned OTEL destinations +(``LLMCallEvent.otel_destinations``, resolved server-side from named credentials). +Its spans must export to ALL of them plus the configured/global exporter, so +``TenantTracerCache`` builds and caches ``TracerProvider``s that append one +``SpanProcessor`` per destination. The gen-AI span path (``tracers_for``) groups +destinations by their backend-required Resource attributes and builds one provider +per group, because a span carries exactly one Resource (a provider property) and a +backend like Arize selects its project FROM the Resource -- so two Arize projects +each get a correctly-tagged span instead of one last-wins merge. Header-routed +backends declare no Resource attributes, so their destinations stay in one group with +multiple exporters and route by per-exporter auth. With no destinations it hands back +the logger's default tracer (global only). Destinations are never request-derived, so +a caller can neither redirect a trace nor spawn providers. """ from collections import OrderedDict -from typing import Any, Mapping from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import Tracer from litellm._logging import verbose_logger -from litellm.integrations.otel.model.config import OpenTelemetryV2Config -from litellm.integrations.otel.presets import dynamic_otlp_headers +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.model.destination import OtelDestination from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, get_tracer, ) -# Exporter kinds that ignore headers — never rewritten with dynamic credentials. +# Exporter kinds that ignore endpoint/headers — never rewritten with a destination. _NON_OTLP_KINDS = ("console", "in_memory", "inmemory", "memory") -# Cap on distinct credential-scoped providers held at once. ``dynamic_params`` -# can be populated from request metadata, so an unbounded cache lets a caller -# spawn one ``TracerProvider`` (plus its ``BatchSpanProcessor`` background -# thread) per unique credential set and exhaust the proxy. The LRU bound keeps -# the working set of active tenants resident while flushing and shutting down -# evicted providers so their threads are reclaimed. +# Cap on distinct destination-scoped providers held at once. Destinations are +# admin-owned (one per key/team), so this is resource hygiene rather than an +# anti-abuse bound: it keeps the working set of active tenants resident while +# flushing and shutting down evicted providers so their exporter threads are +# reclaimed. _MAX_CACHED_PROVIDERS = 256 @@ -49,7 +54,7 @@ def _shutdown_provider(provider: TracerProvider) -> None: class TenantTracerCache: - """Credential-scoped ``TracerProvider`` cache keyed by the dynamic headers.""" + """Destination-scoped ``TracerProvider`` cache keyed by endpoint + headers.""" def __init__( self, @@ -60,49 +65,162 @@ class TenantTracerCache: self._config = config self._callback_name = callback_name self._tracer_name = tracer_name - self._providers: "OrderedDict[tuple[tuple[str, str], ...], TracerProvider]" = OrderedDict() + self._providers: OrderedDict[tuple[object, ...], TracerProvider] = OrderedDict() - def tracer_for(self, default: Tracer, dynamic_params: Any) -> Tracer: - """Return the tracer for this request. + def tracers_for(self, default: Tracer, destinations: "tuple[OtelDestination, ...]") -> "tuple[Tracer, ...]": + """The tracers for this request's gen-AI span, one per distinct Resource group. - Use ``default`` unless the request's dynamic credentials require a - credential-scoped tracer, in which case build (or reuse) one. The cache - is a bounded LRU: the least-recently-used provider is flushed and shut - down on overflow so its exporter threads don't accumulate. + A span carries exactly one Resource (it's a property of the ``TracerProvider``), + but a backend like Arize selects its project FROM the Resource + (``arize.project.name`` / ``model_id``), so two Arize destinations with different + projects need two differently-tagged spans. Group the backend's resolved + destinations by ``destination_resource_attrs`` and return one tracer per group; + the caller emits the span once per tracer (mirroring how the fan-out processor + re-wraps proxy-internal spans per destination). + + Header-routed backends (langfuse, weave) declare no Resource attributes, so all + their destinations collapse into one empty-Resource group with one exporter each + and keep routing by per-exporter auth -- unchanged from the single-group path. + The configured/global exporters ride the FIRST group only, so the global receives + the span once. Empty ``destinations`` -> the logger's default tracer (deny). """ - headers = dynamic_otlp_headers(self._callback_name, dynamic_params) - if not headers: - return default - cache_key = tuple(sorted(headers.items())) + if not destinations: + return (default,) + return tuple( + self._tracer_for_group(resource_key, group, include_base=index == 0) + for index, (resource_key, group) in enumerate(self._group_by_resource(destinations)) + ) + + def _group_by_resource( + self, destinations: "tuple[OtelDestination, ...]" + ) -> "list[tuple[tuple[tuple[str, str], ...], list[OtelDestination]]]": + """Destinations grouped by their backend-required Resource attributes. + + The key is a stable sorted tuple of ``destination_resource_attrs`` items. + Groups are returned in a deterministic order (sorted by key), so the + empty-Resource group (header-routed backends) sorts first and the + configured/global exporters attach to it. + """ + from litellm.integrations.otel.plumbing.providers import ( + destination_resource_attrs, + ) + + groups: OrderedDict[tuple[tuple[str, str], ...], list[OtelDestination]] = OrderedDict() + for destination in destinations: + key = tuple(sorted(destination_resource_attrs(destination).items())) + groups.setdefault(key, []).append(destination) + return sorted(groups.items()) + + def _tracer_for_group( + self, + resource_key: "tuple[tuple[str, str], ...]", + group: "list[OtelDestination]", + *, + include_base: bool, + ) -> Tracer: + cache_key: tuple[object, ...] = ( + resource_key, + tuple(sorted((d.endpoint, tuple(sorted(d.headers.items()))) for d in group)), + include_base, + ) provider = self._providers.get(cache_key) if provider is not None: self._providers.move_to_end(cache_key) else: - provider = build_tracer_provider(self._config_with_headers(headers)) + provider = build_tracer_provider( + self._config_with_destinations(tuple(group), include_base_exporters=include_base) + ) self._providers[cache_key] = provider if len(self._providers) > _MAX_CACHED_PROVIDERS: _, evicted = self._providers.popitem(last=False) _shutdown_provider(evicted) return get_tracer(provider, self._tracer_name) - def _config_with_headers(self, headers: Mapping[str, str]) -> OpenTelemetryV2Config: - """Clone the config, stamping ``headers`` onto the credential's own exporter. + def tracer_for(self, default: Tracer, destinations: "tuple[OtelDestination, ...]") -> Tracer: + """Single merged tracer for ``destinations`` (one provider, one Resource). - ``headers`` are the per-request credentials of ``self._callback_name`` (the - integration that built this cache), so they apply only to the exporter that - integration contributed (``spec.owner``). A request that carries one - tenant's Arize key must never rewrite the headers of a co-configured - Langfuse or self-hosted collector exporter, which would leak that key to a - different backend. + The single-group primitive: kept for the destination-set cache mechanics and as + the building block ``tracers_for`` composes per group. The gen-AI span path uses + ``tracers_for`` so multiple Resource groups aren't last-wins merged. """ - header_str = ",".join(f"{key}={value}" for key, value in headers.items()) - header_update: dict[str, str] = {"headers": header_str} - exporters = [ - ( - spec.model_copy(update=header_update) - if spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS - else spec + if not destinations: + return default + cache_key = tuple(sorted((d.endpoint, tuple(sorted(d.headers.items()))) for d in destinations)) + provider = self._providers.get(cache_key) + if provider is not None: + self._providers.move_to_end(cache_key) + else: + provider = build_tracer_provider(self._config_with_destinations(destinations)) + self._providers[cache_key] = provider + if len(self._providers) > _MAX_CACHED_PROVIDERS: + _, evicted = self._providers.popitem(last=False) + _shutdown_provider(evicted) + return get_tracer(provider, self._tracer_name) + + def _owned_otlp_kind(self) -> str: + """The OTLP transport of this integration's own exporter (langfuse -> http, + arize -> grpc), used for the destinations appended below. + + Prefer the admin's configured exporter kind for this backend; fall back to + the backend's intrinsic default (shared with the fan-out processor via + ``default_otlp_kind_for_backend``) so a lazily-activated backend with no + owned spec still picks the right transport (e.g. arize -> grpc, not http). + """ + from litellm.integrations.otel.plumbing.providers import ( + default_otlp_kind_for_backend, + ) + + for spec in self._config.exporters: + if spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS: + return spec.kind + return default_otlp_kind_for_backend(self._callback_name) + + def _config_with_destinations( + self, + destinations: "tuple[OtelDestination, ...]", + *, + include_base_exporters: bool = True, + ) -> OpenTelemetryV2Config: + """Clone the config and APPEND one exporter per resolved destination. The shared + ``TracerProvider`` attaches one ``SpanProcessor`` per spec, so a single span is + emitted once and exported to every appended destination. Each appended exporter's + endpoint is the resolved host (the cross-host fix) with its own auth headers + (per-destination isolation). + + ``include_base_exporters`` keeps the configured/global exporters too (so the + global still receives). ``tracers_for`` sets it only on the first Resource group, + so when a backend splits into multiple groups the global gets the span once + rather than once per group. + + The clone's Resource folds in the destinations' backend-required Resource + attributes (Arize needs ``model_id`` / ``arize.project.name``), via the same + ``destination_resource_attrs`` the fan-out path uses on proxy-internal spans. + Callers group destinations by those attributes first, so within one call all + ``destinations`` share a Resource and the merge is not lossy -- without this the + gen-AI span would reach Arize with only ``service.name`` while its parents + (fan-out) carry ``model_id``, orphaning the subtree.""" + from litellm.integrations.otel.plumbing.providers import ( + destination_resource_attrs, + ) + + kind = self._owned_otlp_kind() + appended = [ + ExporterSpec( + kind=kind, + endpoint=d.endpoint, + headers=d.header_string(), + owner=None, ) - for spec in self._config.exporters + for d in destinations ] - return self._config.model_copy(update={"exporters": exporters}) + base_exporters = [*self._config.exporters] if include_base_exporters else [] + merged_resource_attrs = { + **self._config.resource_attributes, + **{key: value for d in destinations for key, value in destination_resource_attrs(d).items()}, + } + return self._config.model_copy( + update={ + "exporters": [*base_exporters, *appended], + "resource_attributes": merged_resource_attrs, + } + ) diff --git a/litellm/integrations/otel/presets/__init__.py b/litellm/integrations/otel/presets/__init__.py index deaf953ede8..635e0480b77 100644 --- a/litellm/integrations/otel/presets/__init__.py +++ b/litellm/integrations/otel/presets/__init__.py @@ -6,22 +6,23 @@ vocabularies to apply, and any resource attributes. ``PRESET_BY_CALLBACK`` maps a callback name (``"arize"``, ``"langfuse_otel"``, ...) to its preset so the factory in ``litellm_logging`` can resolve a name and build a single ``OpenTelemetryV2`` instance from the result. + +Per-key/team routing does not live here. A trace destination is admin-owned +infrastructure config, resolved server-side from a named credential into an +``OtelDestination`` (see ``litellm.integrations.otel.presets.destinations`` and +``plumbing.routing``); nothing in this package reads vendor credentials or a +host off a request. """ -from typing import Callable - from litellm.integrations.otel.presets.agentops import agentops_preset -from litellm.integrations.otel.presets.arize import arize_dynamic_headers, arize_preset +from litellm.integrations.otel.presets.arize import arize_preset from litellm.integrations.otel.presets.base import Preset -from litellm.integrations.otel.presets.langfuse import ( - langfuse_dynamic_headers, - langfuse_preset, -) +from litellm.integrations.otel.presets.generic import generic_preset +from litellm.integrations.otel.presets.langfuse import langfuse_preset from litellm.integrations.otel.presets.langtrace import langtrace_preset from litellm.integrations.otel.presets.levo import levo_preset from litellm.integrations.otel.presets.phoenix import phoenix_preset -from litellm.integrations.otel.presets.weave import weave_dynamic_headers, weave_preset -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.integrations.otel.presets.weave import weave_preset #: Callback name → preset. The ``Preset`` annotation makes mypy verify every #: registered value matches the preset interface. @@ -29,45 +30,20 @@ PRESET_BY_CALLBACK: dict[str, Preset] = { "agentops": agentops_preset, "arize": arize_preset, "arize_phoenix": phoenix_preset, + "generic": generic_preset, "langfuse_otel": langfuse_preset, "langtrace": langtrace_preset, "levo": levo_preset, "weave_otel": weave_preset, } -#: Callback name → per-request OTLP header builder (team/key multi-tenant -#: routing). Only integrations that support dynamic credentials appear here — -#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's -#: default tracer. -DYNAMIC_HEADERS_BY_CALLBACK: dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]] = { - "arize": arize_dynamic_headers, - "langfuse_otel": langfuse_dynamic_headers, - "weave_otel": weave_dynamic_headers, -} - - -def dynamic_otlp_headers( - callback_name: str | None, - dynamic_params: StandardCallbackDynamicParams | None, -) -> dict[str, str] | None: - """Per-request OTLP headers for ``callback_name``, or ``None`` if N/A. - - ``None`` means "no per-request routing" — the caller uses its default tracer. - """ - builder = DYNAMIC_HEADERS_BY_CALLBACK.get(callback_name or "") - if builder is None or not dynamic_params: - return None - headers = builder(dynamic_params) - return headers or None - __all__ = [ "PRESET_BY_CALLBACK", - "DYNAMIC_HEADERS_BY_CALLBACK", "Preset", - "dynamic_otlp_headers", "agentops_preset", "arize_preset", + "generic_preset", "langfuse_preset", "langtrace_preset", "levo_preset", diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index 048b63c89fb..94568c4122c 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -39,6 +39,7 @@ class _AgentOpsSettings(BaseSettings): def agentops_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Build the AgentOps config without any network I/O. @@ -49,17 +50,24 @@ def agentops_preset( """ settings = _AgentOpsSettings() base = config_overrides or OpenTelemetryV2Config() + # Contribute the global AgentOps exporter only when an API key is configured. + # Without it the lazy-auth exporter has nothing to mint a JWT from and every + # export fails; admin-owned destinations carry their own credentials. + global_exporter = ( + ( + ExporterSpec( + kind=_AGENTOPS_EXPORTER_KIND, + endpoint=_AGENTOPS_ENDPOINT, + options={"api_key": settings.api_key}, + owner=ExporterOwner.AGENTOPS, + ), + ) + if settings.api_key + else () + ) return base.model_copy( update={ - "exporters": [ - *base.exporters, - ExporterSpec( - kind=_AGENTOPS_EXPORTER_KIND, - endpoint=_AGENTOPS_ENDPOINT, - options=({"api_key": settings.api_key} if settings.api_key else None), - owner=ExporterOwner.AGENTOPS, - ), - ], + "exporters": [*base.exporters, *global_exporter], "resource_attributes": { **base.resource_attributes, "service.name": settings.service_name, diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py index 95206205630..7b58234bd2a 100644 --- a/litellm/integrations/otel/presets/arize.py +++ b/litellm/integrations/otel/presets/arize.py @@ -10,7 +10,6 @@ from litellm.integrations.otel.model.config import ( OpenTelemetryV2Config, ) from litellm.integrations.otel.presets.utils import ensure_mappers -from litellm.types.utils import StandardCallbackDynamicParams class _ArizeSettings(BaseSettings): @@ -24,21 +23,30 @@ class _ArizeSettings(BaseSettings): def arize_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: arize_cfg = _V1ArizeLogger.get_arize_config() headers = _arize_headers(arize_cfg) base = config_overrides or OpenTelemetryV2Config() + # Contribute the global Arize exporter only when Arize credentials are + # configured. Without them it points at the Arize cloud with no auth and every + # export fails PERMISSION_DENIED; admin-owned destinations carry their own + # credentials and are appended by the router instead. + global_exporter = ( + ( + ExporterSpec( + kind=arize_cfg.protocol or "otlp_grpc", + endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1", + headers=headers, + owner=ExporterOwner.ARIZE_AX, + ), + ) + if headers + else () + ) return base.model_copy( update={ - "exporters": [ - *base.exporters, - ExporterSpec( - kind=arize_cfg.protocol or "otlp_grpc", - endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1", - headers=headers, - owner=ExporterOwner.ARIZE_AX, - ), - ], + "exporters": [*base.exporters, *global_exporter], "mapper_names": ensure_mappers(base.mapper_names, "openinference"), "resource_attributes": { **base.resource_attributes, @@ -59,16 +67,3 @@ def _arize_headers(arize_cfg) -> str | None: # credentials are configured. return _ArizeSettings().otlp_traces_headers return ",".join(pieces) - - -def arize_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: - """Per-request Arize OTLP headers from team/key dynamic params.""" - headers: dict[str, str] = {} - # ``arize_space_key`` is the suggested param and wins over ``arize_space_id``. - space = params.get("arize_space_key") or params.get("arize_space_id") - if space: - headers["arize-space-id"] = space - api_key = params.get("arize_api_key") - if api_key: - headers["api_key"] = api_key - return headers diff --git a/litellm/integrations/otel/presets/base.py b/litellm/integrations/otel/presets/base.py index 3b9991f86a4..18628290299 100644 --- a/litellm/integrations/otel/presets/base.py +++ b/litellm/integrations/otel/presets/base.py @@ -18,6 +18,20 @@ class Preset(Protocol): ``config_overrides`` lets one preset layer onto another's config (or onto test-supplied defaults); the factory calls presets with no arguments. + + ``allow_missing_credentials`` distinguishes the two reasons a preset is built. + A credential-mandatory backend (weave/langfuse/levo) raises when its global env + credentials are absent so a misconfigured global callback fails loud at startup; + set this when the only reason for construction is an admin-owned destination, + which carries its own per-tenant credentials, so the preset degrades to an + exporter-less (mapper-only) config instead of raising. Credential-optional + backends (arize/phoenix/agentops/langtrace) already run without a global + exporter and ignore it. """ - def __call__(self, *, config_overrides: OpenTelemetryV2Config | None = None) -> OpenTelemetryV2Config: ... + def __call__( + self, + *, + config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, + ) -> OpenTelemetryV2Config: ... diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py new file mode 100644 index 00000000000..ad099989537 --- /dev/null +++ b/litellm/integrations/otel/presets/destinations.py @@ -0,0 +1,130 @@ +"""Resolve an admin-owned named credential into a typed OTLP destination. + +The destination (endpoint + auth headers) is admin infrastructure config. Each +OTEL backend stores its own fields on the named credential's free-form +``credential_values``; the adapter here maps those fields to the universal +``OtelDestination`` the v2 router exports through. A backend with no bespoke +adapter is still reachable through the generic ``otel_endpoint`` / ``otel_headers`` +passthrough, so the registry covers every OTEL destination rather than an +enumerated few. Nothing here reads request data; callers pass admin-resolved +credential values only. +""" + +import os +from typing import Callable, Mapping, Optional + +from litellm.constants import LITELLM_LOGGING_CREDENTIAL_NAME_KEY +from litellm.integrations.langfuse.langfuse_otel import ( + LANGFUSE_CLOUD_US_ENDPOINT, + LangfuseOtelLogger, +) +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.weave.weave_otel import _get_weave_authorization_header + +#: Reserved ``callback_vars`` key binding a key/team's logging callback to a named +#: credential in the registry. It is a reference, resolved server-side; it is never +#: forwarded as a request parameter. +LOGGING_CREDENTIAL_NAME_KEY = LITELLM_LOGGING_CREDENTIAL_NAME_KEY + + +def _parse_header_string(raw: str) -> dict[str, str]: + pairs = (item.split("=", 1) for item in raw.split(",") if "=" in item) + return {key.strip(): value.strip() for key, value in pairs} + + +def _langfuse_endpoint(host: str) -> str: + normalized = host if host.startswith("http") else f"https://{host}" + return f"{normalized.rstrip('/')}/api/public/otel" + + +def _langfuse_destination(values: Mapping[str, str]) -> Optional[OtelDestination]: + public_key = values.get("langfuse_public_key") + secret_key = values.get("langfuse_secret_key") + if not public_key or not secret_key: + return None + host = values.get("langfuse_host") + endpoint = _langfuse_endpoint(host) if host else LANGFUSE_CLOUD_US_ENDPOINT + auth = LangfuseOtelLogger._get_langfuse_authorization_header(public_key=public_key, secret_key=secret_key) + return OtelDestination(endpoint=endpoint, headers={"Authorization": auth}) + + +def _arize_destination(values: Mapping[str, str]) -> Optional[OtelDestination]: + space = values.get("arize_space_id") or values.get("arize_space_key") + api_key = values.get("arize_api_key") + if not space or not api_key: + return None + endpoint = values.get("arize_endpoint") or "https://otlp.arize.com/v1" + # Arize routes a trace to a project via the ``model_id`` Resource attribute + # (OpenInference convention), NOT an auth header like langfuse/weave do, so the + # project must ride the span Resource. Prefer the credential's own project, then + # fall back to the proxy-global ``ARIZE_PROJECT_NAME`` so an arize credential + # that omits the project still lands somewhere deterministic. Backends that route + # by header declare no resource_attributes; this stays arize-local. + project = values.get("arize_project_name") or values.get("project_name") or os.environ.get("ARIZE_PROJECT_NAME") + resource_attributes = {"model_id": project, "arize.project.name": project} if project else {} + return OtelDestination( + endpoint=endpoint, + headers={"space_id": space, "api_key": api_key}, + resource_attributes=resource_attributes, + ) + + +def _weave_destination(values: Mapping[str, str]) -> Optional[OtelDestination]: + api_key = values.get("wandb_api_key") + if not api_key: + return None + # Weave's OTLP path is ``/otel/v1/traces`` (not the bare ``/v1/traces`` the + # generic exporter would append), so a host like ``https://trace.wandb.ai`` + # must be completed here -- otherwise the export 404s and silently drops. The + # host itself defaults to the Weave cloud base (only dedicated/self-hosted wandb + # differs), so the endpoint is optional. Mirror the v1 integration's + # WEAVE_BASE_URL / WEAVE_OTEL_ENDPOINT and stay idempotent if the caller already + # supplied the full path or the ``/otel`` prefix. + from litellm.integrations.weave.weave_otel import ( + WEAVE_BASE_URL, + WEAVE_OTEL_ENDPOINT, + ) + + base = (values.get("weave_endpoint") or WEAVE_BASE_URL).rstrip("/") + endpoint = base if base.endswith("/v1/traces") else base.removesuffix("/otel") + WEAVE_OTEL_ENDPOINT + headers = {"Authorization": _get_weave_authorization_header(api_key=api_key)} + project_id = values.get("weave_project_id") + if project_id: + headers["project_id"] = project_id + return OtelDestination(endpoint=endpoint, headers=headers) + + +def _generic_destination(values: Mapping[str, str]) -> Optional[OtelDestination]: + """Any OTLP backend: an explicit endpoint plus raw headers. The catch-all that + makes the registry cover self-hosted collectors / Phoenix / Honeycomb / etc.""" + endpoint = values.get("otel_endpoint") + if not endpoint: + return None + return OtelDestination(endpoint=endpoint, headers=_parse_header_string(values.get("otel_headers", ""))) + + +_ADAPTERS: dict[str, Callable[[Mapping[str, str]], Optional[OtelDestination]]] = { + "langfuse_otel": _langfuse_destination, + "arize": _arize_destination, + "weave_otel": _weave_destination, +} + +#: OTEL v2 callbacks that can be routed to a per-key/team admin destination. +OTEL_V2_DESTINATION_CALLBACKS = frozenset(_ADAPTERS) + + +def build_destination(callback_name: str, values: Mapping[str, str]) -> Optional[OtelDestination]: + """Map an admin credential's ``values`` to an ``OtelDestination`` for + ``callback_name``, falling back to the generic OTLP passthrough. + + Values are trimmed first: a stray leading/trailing space in an endpoint or + host (an easy slip in the create form) yields a malformed OTLP URL the + exporter rejects with a 404, so whitespace is never significant here. + """ + trimmed = {key: value.strip() if isinstance(value, str) else value for key, value in values.items()} + adapter = _ADAPTERS.get(callback_name) + if adapter is not None: + destination = adapter(trimmed) + if destination is not None: + return destination + return _generic_destination(trimmed) diff --git a/litellm/integrations/otel/presets/generic.py b/litellm/integrations/otel/presets/generic.py new file mode 100644 index 00000000000..501cccaf864 --- /dev/null +++ b/litellm/integrations/otel/presets/generic.py @@ -0,0 +1,21 @@ +"""Generic OTLP passthrough preset. + +A ``generic`` admin-owned destination carries only an ``otel_endpoint`` / +``otel_headers`` (no vendor adapter), so this preset is deliberately vendor-neutral: +it yields a base ``OpenTelemetryV2`` config with the standard ``gen_ai.*`` (+ ``legacy``) +span vocabulary, NO vendor mapper, and NO exporter of its own. The per-destination +exporter is appended by ``TenantTracerCache`` from the resolved ``OtelDestination``, so a +generic destination receives the same complete trace tree as Arize/Langfuse/Weave -- +root server span, auth/db internal spans, the ``chat `` gen-AI span, and +cost/error spans. It needs no global OTEL env vars and never raises. +""" + +from litellm.integrations.otel.model.config import OpenTelemetryV2Config + + +def generic_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, +) -> OpenTelemetryV2Config: + return config_overrides or OpenTelemetryV2Config() diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index 5631da6429f..b2b5442409b 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -9,16 +9,27 @@ from litellm.integrations.otel.model.config import ( OpenTelemetryV2Config, ) from litellm.integrations.otel.presets.utils import ensure_mappers -from litellm.types.utils import StandardCallbackDynamicParams def langfuse_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - cfg = _V1Langfuse.get_langfuse_otel_config() - kind = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" base = config_overrides or OpenTelemetryV2Config() + mappers = ensure_mappers(base.mapper_names, "langfuse") + # ``get_langfuse_otel_config()`` raises without Langfuse keys. Propagate that raise + # for a global callback so a misconfigured deployment fails loud, but when an + # admin-owned Langfuse destination is the reason for construction it carries its own + # per-tenant keys, so degrade to a (global-exporter-less) mapper-only config -- or + # the gen-AI span falls to the generic logger and never reaches it. + try: + cfg = _V1Langfuse.get_langfuse_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy(update={"mapper_names": mappers}) + kind = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" return base.model_copy( update={ "exporters": [ @@ -30,19 +41,6 @@ def langfuse_preset( owner=ExporterOwner.LANGFUSE_OTEL, ), ], - "mapper_names": ensure_mappers(base.mapper_names, "langfuse"), + "mapper_names": mappers, } ) - - -def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: - """Per-request Langfuse OTLP headers from team/key dynamic params.""" - public_key = params.get("langfuse_public_key") - secret_key = params.get("langfuse_secret_key") - if public_key and secret_key: - return { - "Authorization": _V1Langfuse._get_langfuse_authorization_header( - public_key=public_key, secret_key=secret_key - ) - } - return {} diff --git a/litellm/integrations/otel/presets/langtrace.py b/litellm/integrations/otel/presets/langtrace.py index acdbaf870d3..e06d69e239c 100644 --- a/litellm/integrations/otel/presets/langtrace.py +++ b/litellm/integrations/otel/presets/langtrace.py @@ -7,6 +7,7 @@ from litellm.integrations.otel.presets.utils import ensure_mappers def langtrace_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Compose the Langtrace mapper on top of the customer's OTLP destination. diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py index 74a95b100cb..eeeadc9d961 100644 --- a/litellm/integrations/otel/presets/levo.py +++ b/litellm/integrations/otel/presets/levo.py @@ -11,9 +11,19 @@ from litellm.integrations.otel.model.config import ( def levo_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - cfg = _V1Levo.get_levo_config() base = config_overrides or OpenTelemetryV2Config() + # ``get_levo_config()`` raises without Levo credentials. Propagate that raise for a + # global callback so a misconfigured deployment fails loud, but when an admin-owned + # Levo destination is the reason for construction it carries its own per-tenant + # credentials, so degrade to a global-exporter-less config rather than raising. + try: + cfg = _V1Levo.get_levo_config() + except Exception: + if not allow_missing_credentials: + raise + return base return base.model_copy( update={ "exporters": [ diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py index 4e7be2c0f51..ecd055cbac4 100644 --- a/litellm/integrations/otel/presets/phoenix.py +++ b/litellm/integrations/otel/presets/phoenix.py @@ -1,5 +1,7 @@ """Arize-Phoenix preset.""" +import os + from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -23,25 +25,40 @@ class _PhoenixSettings(BaseSettings): ) +_PHOENIX_ENV_VARS = ( + "PHOENIX_API_KEY", + "PHOENIX_COLLECTOR_ENDPOINT", + "PHOENIX_COLLECTOR_HTTP_ENDPOINT", +) + + def phoenix_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - cfg = _V1Phoenix.get_arize_phoenix_config() - headers = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None project_name = _PhoenixSettings().project_name base = config_overrides or OpenTelemetryV2Config() + # Contribute the global Phoenix exporter only when Phoenix is configured (a + # cloud API key or a collector endpoint). Otherwise the config defaults to + # http://localhost:6006 and would export there even when the operator only + # uses admin-owned Phoenix destinations. + if any(os.environ.get(v) for v in _PHOENIX_ENV_VARS): + cfg = _V1Phoenix.get_arize_phoenix_config() + headers = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None + global_exporter = ( + ExporterSpec( + kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http", + endpoint=cfg.endpoint, + headers=headers, + owner=ExporterOwner.ARIZE_PHOENIX, + ), + ) + else: + global_exporter = () return base.model_copy( update={ - "exporters": [ - *base.exporters, - ExporterSpec( - kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http", - endpoint=cfg.endpoint, - headers=headers, - owner=ExporterOwner.ARIZE_PHOENIX, - ), - ], + "exporters": [*base.exporters, *global_exporter], "mapper_names": ensure_mappers(base.mapper_names, "openinference"), "resource_attributes": { **base.resource_attributes, diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py index d22f7641289..4c149b2ec2b 100644 --- a/litellm/integrations/otel/presets/weave.py +++ b/litellm/integrations/otel/presets/weave.py @@ -6,19 +6,28 @@ from litellm.integrations.otel.model.config import ( OpenTelemetryV2Config, ) from litellm.integrations.otel.presets.utils import ensure_mappers -from litellm.integrations.weave.weave_otel import ( - _get_weave_authorization_header, - get_weave_otel_config, -) -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.integrations.weave.weave_otel import get_weave_otel_config def weave_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - weave_cfg = get_weave_otel_config() base = config_overrides or OpenTelemetryV2Config() + # Weave consumes OpenInference + a small Weave-specific overlay. + mappers = ensure_mappers(base.mapper_names, "openinference", "weave") + # ``get_weave_otel_config()`` raises without W&B credentials. Propagate that raise + # for a global callback so a misconfigured deployment fails loud, but when an + # admin-owned Weave destination is the reason for construction it carries its own + # per-tenant credentials, so degrade to a (global-exporter-less) mapper-only config + # -- otherwise the gen-AI span falls to the generic logger and never reaches Weave. + try: + weave_cfg = get_weave_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy(update={"mapper_names": mappers}) return base.model_copy( update={ "exporters": [ @@ -30,19 +39,6 @@ def weave_preset( owner=ExporterOwner.WEAVE_OTEL, ), ], - # Weave consumes OpenInference + a small Weave-specific overlay. - "mapper_names": ensure_mappers(base.mapper_names, "openinference", "weave"), + "mapper_names": mappers, } ) - - -def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: - """Per-request Weave OTLP headers from team/key dynamic params.""" - headers: dict[str, str] = {} - api_key = params.get("wandb_api_key") - if api_key: - headers["Authorization"] = _get_weave_authorization_header(api_key=api_key) - project_id = params.get("weave_project_id") - if project_id: - headers["project_id"] = project_id - return headers diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index d0a6ec30c9e..771fba451de 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,6 +1,6 @@ -from typing import Dict, Optional +from typing import Dict, Optional, cast -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import OtelDestinationParams, StandardCallbackDynamicParams def _is_env_reference(value: object) -> bool: @@ -102,4 +102,19 @@ def initialize_standard_callback_dynamic_params( validate_no_callback_env_reference(param, _param_value, source="metadata") standard_callback_dynamic_params[param] = _param_value # type: ignore + # Admin-owned OTEL v2 destinations, resolved server-side by the proxy from the + # exporters assigned to the request's identity chain. The proxy stamps them + # onto ``data["litellm_metadata"]["otel_destinations"]`` -- ``litellm_metadata`` + # is in ``all_litellm_params``, so it is scrubbed from the body before it reaches + # the provider. ``otel_destinations`` is intentionally NOT a top-level key so an + # unknown field cannot leak to provider APIs. Read from ``litellm_metadata`` only, + # never from request ``metadata``, and never via ``_supported_callback_params``, + # so a request body cannot set or select a trace destination. + proxy_metadata = kwargs.get("litellm_metadata") or {} + otel_destinations = proxy_metadata.get("otel_destinations") if isinstance(proxy_metadata, dict) else None + if isinstance(otel_destinations, list): + standard_callback_dynamic_params["otel_destinations"] = cast( + "list[OtelDestinationParams]", otel_destinations + ) + return standard_callback_dynamic_params diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2457d117b81..04f26d3babf 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3888,9 +3888,9 @@ def _init_custom_logger_compatible_class( otel_config = OpenTelemetryConfig( exporter="otlp_http", - endpoint="https://langtrace.ai/api/trace", + endpoint="https://app.langtrace.ai/api/trace", ) - os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = f"api_key={os.getenv('LANGTRACE_API_KEY')}" + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = f"x-api-key={os.getenv('LANGTRACE_API_KEY')}" for callback in _in_memory_loggers: if isinstance(callback, OpenTelemetry) and callback.callback_name == "langtrace": return callback # type: ignore @@ -3952,6 +3952,13 @@ def _init_custom_logger_compatible_class( _otel_logger = WeaveOtelLogger(config=otel_config, callback_name="weave_otel") _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore + elif logging_integration == "generic": + # Generic OTLP passthrough: a vendor-neutral OpenTelemetryV2 logger whose + # per-destination exporter is attached by the admin-owned destination, so a + # ``generic`` destination gets the full trace (incl. the gen-AI span), not + # just proxy-internal spans. Only meaningful as an admin-owned destination, + # so there is no legacy fallback: None when no v2 logger is constructed. + return _maybe_construct_otel_v2("generic", _in_memory_loggers) # type: ignore elif logging_integration == "pagerduty": for callback in _in_memory_loggers: if isinstance(callback, PagerDutyAlerting): @@ -4076,16 +4083,39 @@ def _init_custom_logger_compatible_class( return None -def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list) -> Optional[Any]: - """If ``LITELLM_OTEL_V2`` is on, build (or reuse) a single ``OpenTelemetryV2`` - instance configured via the preset for ``callback_name``. +def _has_admin_owned_logging_destination(callback_name: str) -> bool: + """Whether an admin has registered a logging destination for this backend. - Returns ``None`` when V2 is off OR when there's no preset registered for + Admin-owned trace destinations (``logging`` credentials, created from the UI) + are an OTEL v2 feature: the v2 logger fans a request's spans out to each + destination using that destination's own credentials. So when one exists for + ``callback_name`` the v2 logger must own the backend even if the global + ``LITELLM_OTEL_V2`` flag is off, otherwise activation falls back to the legacy + global logger, which ignores the per-destination credentials and exports with + whatever (often absent) global env credentials are set. + """ + import litellm + + return any( + (info := credential.credential_info or {}).get("credential_type") == "logging" + and info.get("description") == callback_name + for credential in litellm.credential_list + ) + + +def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list) -> Optional[Any]: + """Build (or reuse) a single ``OpenTelemetryV2`` instance configured via the + preset for ``callback_name`` when V2 owns this backend. + + V2 owns the backend when the global ``LITELLM_OTEL_V2`` flag is on, or when an + admin-owned logging destination is registered for it (which is itself a V2-only + feature). Returns ``None`` otherwise, or when there's no preset registered for ``callback_name`` — callers should then fall through to the legacy path. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled - if not is_otel_v2_enabled(): + has_admin_dest = _has_admin_owned_logging_destination(callback_name) + if not is_otel_v2_enabled() and not has_admin_dest: return None from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.presets import PRESET_BY_CALLBACK @@ -4097,10 +4127,13 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list) -> Op if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == callback_name: return callback try: - config = preset_fn() + # An admin-owned destination carries its own per-tenant credentials, so a + # credential-mandatory preset may degrade rather than raise. A purely global + # callback with no destination must still raise on missing credentials; the + # raise is swallowed here so the caller defers to the legacy path and customers + # get the same loud error story they had before V2 landed. + config = preset_fn(allow_missing_credentials=has_admin_dest) except Exception: - # If env vars are missing or the preset raises, defer to the legacy path - # so customers get the same error story they had before V2 landed. return None v2_logger = OpenTelemetryV2(config=config, callback_name=callback_name) _in_memory_loggers.append(v2_logger) @@ -4271,6 +4304,12 @@ def get_custom_logger_compatible_class( for callback in _in_memory_loggers: if isinstance(callback, ArizeLogger) and callback.callback_name == "arize": return callback + elif logging_integration == "generic": + from litellm.integrations.otel.logger import OpenTelemetryV2 + + for callback in _in_memory_loggers: + if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == "generic": + return callback elif logging_integration == "logfire": if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") diff --git a/litellm/models/credentials.py b/litellm/models/credentials.py index b74ea055d21..cdad7fdc24c 100644 --- a/litellm/models/credentials.py +++ b/litellm/models/credentials.py @@ -7,7 +7,7 @@ layer; ``litellm.types.utils`` re-exports them for backwards compatibility. from typing import Optional -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator class CredentialBase(BaseModel): @@ -29,3 +29,51 @@ class CreateCredentialItem(CredentialBase): if not values.get("credential_values") and not values.get("model_id"): raise ValueError("Either credential_values or model_id must be set") return values + + +class UpdateCredentialItem(BaseModel): + """PATCH body for ``/credentials/{name}``. + + Both ``credential_values`` and ``credential_info`` are optional so a caller + can patch one without sending the other (team-admins patching access without + knowing the upstream secrets; proxy admins rotating values without touching + access). ``credential_name`` is optional because most patches don't rename. + """ + + credential_name: Optional[str] = None + credential_values: Optional[dict] = None + credential_info: Optional[dict] = None + + +class CredentialAccess(BaseModel): + """Destination-side access list on a logging credential. + + ``global`` is exposed via the JSON name "global" through a field alias since + that's a Python keyword. ``populate_by_name`` keeps internal Python code + using ``global_`` working while JSON in/out uses "global". + """ + + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + global_: bool = Field(default=False, alias="global") + teams: tuple[str, ...] = () + orgs: tuple[str, ...] = () + + +class CredentialInfo(BaseModel): + """Typed shape of ``credential_info`` for the access-control decider. + + Existing stored credentials carry arbitrary extra fields (e.g. + ``custom_llm_provider``); ``extra="allow"`` preserves them. The decider + inspects ``model_fields_set`` to learn which fields the caller actually + patched, which is what Pydantic gives us natively without dict-key spelunking. + """ + + model_config = ConfigDict(extra="allow") + + credential_type: Optional[str] = None + description: Optional[str] = None + host: Optional[str] = None + endpoint: Optional[str] = None + access: Optional[CredentialAccess] = None + auto_enable: bool = False diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d84588a4c24..b2250c9fbc7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -16,7 +16,10 @@ from pydantic import ( from typing_extensions import Required, TypedDict from litellm._uuid import uuid -from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS +from litellm.constants import ( + LITELLM_LOGGING_CREDENTIAL_NAME_KEY, + MCP_STDIO_ALLOWED_COMMANDS, +) from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_no_callback_env_reference, ) @@ -762,6 +765,12 @@ class LiteLLMRoutes(enum.Enum): # Team guardrail submissions - endpoint scopes results to caller's teams (non-admin) "/guardrails/submissions", "/guardrails/submissions/{guardrail_id}", + # Logging-credential routes. GET filters to logging-typed for non-admins; + # PATCH delegates to decide_credential_patch in credential_endpoints, which + # only allows a team-admin to append their own team_id to access.teams. + # POST and DELETE stay proxy-admin only via is_admin_gated_credential_info. + "/credentials", + "/credentials/{credential_name}", ] # routes that manage their own allowed/disallowed logic ## Org Admin Routes ## @@ -1825,7 +1834,7 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): @classmethod def validate_callback_vars(cls, values): callback_vars = values.get("callback_vars", {}) - valid_keys = set(StandardCallbackDynamicParams.__annotations__.keys()) + valid_keys = set(StandardCallbackDynamicParams.__annotations__.keys()) | {LITELLM_LOGGING_CREDENTIAL_NAME_KEY} for key, value in callback_vars.items(): if key not in valid_keys: raise ValueError(f"Invalid callback variable: {key}. Must be one of {valid_keys}") @@ -1864,7 +1873,7 @@ class TeamCallbackMetadata(LiteLLMPydanticObjectBase): "callbacks": [], "callback_vars": {}, } - valid_keys = set(StandardCallbackDynamicParams.__annotations__.keys()) + valid_keys = set(StandardCallbackDynamicParams.__annotations__.keys()) | {LITELLM_LOGGING_CREDENTIAL_NAME_KEY} if callback_vars is not None: for key in callback_vars: if key not in valid_keys: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 73f9def822b..3e351d2ef6d 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -911,6 +911,46 @@ async def _resolve_jwt_to_virtual_key( return None +async def _hoist_request_destinations(request: Request, user_api_key_dict: UserAPIKeyAuth) -> None: + """Resolve admin-owned OTEL destinations for this request and anchor them. + + Runs after the auth builder, while we are still inside the request task, so + the ``ContextVar`` is visible to every ``SpanProcessor.on_end`` that fires + for spans this request opens. Stashes the same list on ``request.state`` so + ``_apply_admin_logging_exporters`` can reuse it without a second DB pass. + + Best-effort: a resolver failure must not break the request. The contextvar + is left at its default (empty tuple), so the fan-out processor no-ops. + """ + try: + from litellm.integrations.otel.model.destination import OtelDestination + from litellm.integrations.otel.plumbing.context import ( + set_request_destinations, + ) + from litellm.proxy.litellm_pre_call_utils import ( + _resolve_logging_exporters, + ) + + destinations_raw, _backends = await _resolve_logging_exporters(user_api_key_dict) + destinations = tuple( + OtelDestination( + callback_name=item.get("callback_name"), + endpoint=item.get("endpoint", ""), + headers=item.get("headers") or {}, + resource_attributes=item.get("resource_attributes") or {}, + ) + for item in destinations_raw + if isinstance(item, dict) and item.get("endpoint") + ) + set_request_destinations(destinations) + try: + request.state.otel_destinations = destinations_raw + except Exception: # noqa: BLE001 + pass + except Exception as exc: # noqa: BLE001 + verbose_proxy_logger.debug("OTel V2: hoist destination resolution failed: %s", exc) + + def _ensure_parent_otel_span_on_request_state(request: Request) -> None: """Idempotently create the OTEL SERVER span and stash it on ``request.state.parent_otel_span``. Safe to call multiple times. @@ -1260,6 +1300,8 @@ async def _user_api_key_auth_builder( valid_token = auto_registered api_key = valid_token.token or "" + await _hoist_request_destinations(request, valid_token) + # Check if model has zero cost - if so, skip all budget checks model = _get_model_from_request_context( request_data=request_data, @@ -1593,6 +1635,8 @@ async def _user_api_key_auth_builder( user_obj: Optional[LiteLLM_UserTable] = None valid_token_dict: dict = {} if valid_token is not None: + valid_token.parent_otel_span = parent_otel_span + await _hoist_request_destinations(request, valid_token) # Got Valid Token from Cache, DB # Run checks for # 1. If token can call model @@ -2422,6 +2466,15 @@ async def user_api_key_auth( ) user_api_key_auth_obj.budget_reservation = None + # Admin-resolved OTEL destinations: anchor them on this request's task + # context BEFORE downstream spans close, so the global-provider fan-out + # processor forwards every span (server, auth, db, batch-write) to the + # admin-assigned per-tenant backends -- not just the gen-AI span the + # ``TenantTracerCache`` already routes. Also stashed on ``request.state`` + # so ``_apply_admin_logging_exporters`` reuses the result instead of + # re-resolving. + await _hoist_request_destinations(request, user_api_key_auth_obj) + ## ENSURE DISABLE ROUTE WORKS ACROSS ALL USER AUTH FLOWS ## RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request) diff --git a/litellm/proxy/credential_endpoints/access_decision.py b/litellm/proxy/credential_endpoints/access_decision.py new file mode 100644 index 00000000000..e5ab909bfe0 --- /dev/null +++ b/litellm/proxy/credential_endpoints/access_decision.py @@ -0,0 +1,132 @@ +"""Pure tagged-union decision for who can patch a logging-credential. + +A logging credential controls where other tenants' traces are exported, so +``credential_info.access`` is the only field a non-admin caller may touch, and +only to add their own team_id(s) to ``access.teams``. Everything else +(values, host, type, description, ``global``, ``orgs``, foreign team_ids, +or removing existing grants) stays proxy-admin only. + +The decision is a value (Allow vs Deny(reason)), kept separate from the +endpoint so it can be unit-tested exhaustively without spinning up FastAPI. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, Mapping + +from litellm.models.credentials import CredentialAccess, CredentialInfo + +OPAQUE_DENY_REASON = "Only the proxy admin can manage logging credentials" + + +@dataclass(frozen=True, slots=True) +class Allow: + tag: Literal["allow"] = "allow" + + +@dataclass(frozen=True, slots=True) +class Deny: + reason: str + # When True the reason is derived from caller input (e.g. they typed a + # foreign team_id) and is safe to surface. When False the reason would + # confirm the stored credential is a logging destination; the endpoint + # collapses these to OPAQUE_DENY_REASON so PATCH /credentials/{name} + # can't be used as an existence oracle by a non-admin caller. + from_user_input: bool = False + tag: Literal["deny"] = "deny" + + +Decision = Allow | Deny + + +_IMMUTABLE_INFO_FIELDS = frozenset({"credential_type", "description", "host", "endpoint"}) + + +def _patched_fields(info: CredentialInfo | None) -> frozenset[str]: + """Names of credential_info fields the caller actually set in their patch.""" + if info is None: + return frozenset() + return frozenset(info.model_fields_set) | frozenset(info.model_extra.keys() if info.model_extra else ()) + + +def _access_teams(access: CredentialAccess | None) -> frozenset[str]: + return frozenset(access.teams) if access is not None else frozenset() + + +def decide_credential_patch( + *, + is_proxy_admin: bool, + caller_team_admin_ids: frozenset[str], + existing_info: CredentialInfo | None, + patch_info: CredentialInfo | None, + patch_values: Mapping[str, object] | None, + patch_name_changed: bool, +) -> Decision: + """Return Allow or Deny(reason) for a PATCH /credentials/{name} request. + + Proxy admins always pass. A team admin only passes when the patch (a) does + not change ``credential_values`` or ``credential_name``, (b) does not + modify any immutable ``credential_info`` field, and (c) limits its + ``access`` change to appending team_ids the caller is team-admin of to + ``access.teams`` (no removals, no foreign ids, no ``global``/``orgs`` + edits). + """ + if is_proxy_admin: + return Allow() + + if not caller_team_admin_ids: + return Deny(OPAQUE_DENY_REASON) + + if patch_name_changed: + return Deny("credential_name is proxy-admin only") + + if patch_values: + return Deny("credential_values is proxy-admin only") + + touched = _patched_fields(patch_info) + if not touched: + return Deny("patch must set credential_info.access for team-admin writes") + + forbidden = touched & _IMMUTABLE_INFO_FIELDS + if forbidden: + return Deny("credential_info fields are proxy-admin only: " + ", ".join(sorted(forbidden))) + + if touched - {"access"}: + return Deny("team-admin may only patch credential_info.access; got: " + ", ".join(sorted(touched))) + + assert patch_info is not None + patch_access = patch_info.access + if patch_access is None: + return Deny("credential_info.access must be set for team-admin writes") + + # Touching global/orgs is allowed when the value matches the stored state + # (the UI's edit modal sends the full access object back so unchecking + # revokes; a no-op resend of global=false / orgs=[] must not be rejected). + # Only block when the caller would actually CHANGE these. + existing_access = existing_info.access if existing_info is not None else None + access_touched = frozenset(patch_access.model_fields_set) + existing_global = existing_access.global_ if existing_access is not None else False + existing_orgs = frozenset(existing_access.orgs) if existing_access is not None else frozenset() + if "global_" in access_touched and patch_access.global_ != existing_global: + return Deny("access.global is proxy-admin only") + if "orgs" in access_touched and frozenset(patch_access.orgs) != existing_orgs: + return Deny("access.orgs is proxy-admin only") + + existing_teams = _access_teams(existing_access) + patch_teams = _access_teams(patch_access) + + foreign_removed = (existing_teams - patch_teams) - caller_team_admin_ids + if foreign_removed: + # Do NOT echo the foreign team_ids -- they're stored values the + # caller didn't send, so naming them would leak access list members. + return Deny("team-admin may only revoke their own team grants") + + foreign_added = (patch_teams - existing_teams) - caller_team_admin_ids + if foreign_added: + return Deny( + "team-admin may only grant their own team_ids: " + ", ".join(sorted(foreign_added)), + from_user_input=True, + ) + + return Allow() diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index e94016c555e..4ce00a44260 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -2,24 +2,164 @@ CRUD endpoints for storing reusable credentials. """ -from typing import Optional +from typing import TYPE_CHECKING, Optional -from fastapi import APIRouter, Depends, HTTPException, Request, Response, Path +from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import _get_masked_values -from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from pydantic import ValidationError + +from litellm.models.credentials import CredentialInfo +from litellm.proxy.credential_endpoints.access_decision import ( + OPAQUE_DENY_REASON, + Allow, + Deny, + decide_credential_patch, +) +from litellm.proxy.management_endpoints.logging_exporter_validation import ( + is_admin_gated_credential_info, + validate_credential_access, +) from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object from litellm.repositories.credentials_repository import CredentialsRepository -from litellm.types.utils import CreateCredentialItem, CredentialItem +from litellm.types.utils import ( + CreateCredentialItem, + CredentialItem, + UpdateCredentialItem, +) router = APIRouter() +def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={"error": "Only the proxy admin can manage logging credentials"}, + ) + + +def _is_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> bool: + return user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN + + +def _summarize_validation_error(ve: ValidationError) -> str: + parts = (".".join(str(loc) for loc in err["loc"]) + ": " + err["msg"] for err in ve.errors()) + return "; ".join(parts) + + +async def _caller_grantable_team_ids( + user_api_key_dict: UserAPIKeyAuth, prisma_client: "Optional[PrismaClient]" +) -> frozenset[str]: + """Team ids the caller may add to / remove from a destination's access.teams. + + Two paths to grantability: + + 1. Direct team-admin: caller is admin of the team (role=admin in + ``members_with_roles``). + 2. Via org-admin: caller is ORG_ADMIN of the team's organization. Org + admins manage every team in their org, even teams they aren't a + direct member of. + + Empty when the caller has no user_id, no DB connection, or admins + nothing. Uses cached ``get_user_object`` / ``get_team_object`` plus one + bounded query for org teams; the role match is done in Python because + ``members_with_roles`` is a JSON column. + """ + if user_api_key_dict.user_id is None or prisma_client is None: + return frozenset() + from litellm.proxy.auth.auth_checks import get_team_object, get_user_object + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + + try: + user_obj = await get_user_object( + user_id=user_api_key_dict.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + if user_obj is None: + return frozenset() + + # Direct team-admin grants: walk the caller's own team list. + team_admin_of: set[str] = set() + for team_id in [tid for tid in (getattr(user_obj, "teams", None) or []) if isinstance(tid, str)]: + team_obj = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + if any( + member.user_id == user_api_key_dict.user_id and member.role == "admin" + for member in (team_obj.members_with_roles or []) + ): + team_admin_of.add(team_id) + + # Org-admin grants: every team in any org the caller admins, even if + # the caller isn't a direct member of that team. + org_admin_of: list[str] = [ + m.organization_id + for m in (user_obj.organization_memberships or []) + if m.organization_id and m.user_role == LitellmUserRoles.ORG_ADMIN.value + ] + org_grantable: set[str] = set() + if org_admin_of: + org_teams = await prisma_client.db.litellm_teamtable.find_many( # type: ignore[union-attr] + where={"organization_id": {"in": org_admin_of}} + ) + org_grantable = {t.team_id for t in org_teams if t.team_id} + + return frozenset(team_admin_of | org_grantable) + except Exception: # noqa: BLE001 + # Best-effort lookup. A miss here means the PATCH decider will deny any + # patch other than a no-op, which is the safe fallback. + verbose_proxy_logger.exception("team-admin lookup failed") + return frozenset() + + +def _credential_in_memory(credential_name: str) -> Optional[CredentialItem]: + return next( + (cred for cred in litellm.credential_list if cred.credential_name == credential_name), + None, + ) + + +async def _credential_for_admin_gate(credential_name: str, prisma_client: object) -> Optional[CredentialItem]: + """Authoritative credential lookup for the admin gate on update/delete. + + The in-process ``litellm.credential_list`` can be stale: a credential created + via the API on another horizontally-scaled instance, or before a restart, + exists only in the DB. Gating on the in-memory copy alone would let a logging + credential that isn't resident be updated/deleted without the proxy-admin + check. Prefer the in-memory copy, fall back to the DB so the gate sees the + real ``credential_info``. + """ + existing = _credential_in_memory(credential_name) + if existing is not None: + return existing + if prisma_client is None: + return None + try: + return await CredentialsRepository( + prisma_client # type: ignore[arg-type] + ).find_by_name(credential_name) + except Exception: # noqa: BLE001 + return None + + class CredentialHelperUtils: @staticmethod def encrypt_credential_values( @@ -57,6 +197,12 @@ async def create_credential( """ from litellm.proxy.proxy_server import llm_router, prisma_client + # POST stays proxy-admin only across the board: route gate was widened so + # team-admins can PATCH access on existing logging destinations, but + # creation of any credential (logging or provider) remains admin-only. + _require_proxy_admin(user_api_key_dict) + validate_credential_access(credential.credential_info) + try: if prisma_client is None: raise HTTPException( @@ -120,17 +266,48 @@ async def get_credentials( ): """ [BETA] endpoint. This might change unexpectedly. + + Proxy admins see every credential (values masked). Team-admins and + org-admins see only logging-typed destinations so they can self-assign + them; provider credentials stay invisible to non-PROXY_ADMINs. Plain + internal users with no team-admin or org-admin status get 403 — they + have no use for the list and shouldn't see destination names, hosts, + or scope metadata (Veria F2). """ + from litellm.proxy.proxy_server import prisma_client + try: + if _is_proxy_admin(user_api_key_dict): + visible = list(litellm.credential_list) + else: + grantable = await _caller_grantable_team_ids(user_api_key_dict, prisma_client) + if not grantable: + raise HTTPException( + status_code=403, + detail={ + "error": ( + "Listing logging destinations requires team-admin or " + "org-admin status. Ask your proxy admin to add you to a " + "team or org." + ) + }, + ) + visible = [ + credential + for credential in litellm.credential_list + if is_admin_gated_credential_info(credential.credential_info) + ] masked_credentials = [ { "credential_name": credential.credential_name, "credential_values": _get_masked_values(credential.credential_values), "credential_info": credential.credential_info, } - for credential in litellm.credential_list + for credential in visible ] return {"success": True, "credentials": masked_credentials} + except HTTPException: + raise except Exception as e: return handle_exception_on_proxy(e) @@ -230,6 +407,10 @@ async def delete_credential( """ from litellm.proxy.proxy_server import prisma_client + # DELETE stays proxy-admin only. The route gate lets team-admins reach + # /credentials/{name} for PATCH; reject any DELETE that isn't proxy-admin. + _require_proxy_admin(user_api_key_dict) + try: if prisma_client is None: raise HTTPException( @@ -274,16 +455,97 @@ def update_db_credential( merged_credential.credential_values.update(encrypted_params) - # update model info + # Merge the patch into the existing credential_info so a partial update (e.g. only + # access.teams) preserves credential_type/description/host AND the untouched + # access subfields (global/orgs/other teams in access). See + # _merge_credential_info for the surgical-access reasoning. if encrypted_credential.credential_info: - """Update credential info""" - if "credential_info" not in merged_credential.credential_info: + if merged_credential.credential_info is None: merged_credential.credential_info = {} - merged_credential.credential_info.update(encrypted_credential.credential_info) + _merge_credential_info(merged_credential.credential_info, encrypted_credential.credential_info) return merged_credential +def _merge_credential_info(into: dict, patch: dict) -> None: + """Merge ``patch`` into ``into`` in place, with surgical access subfields. + + A prior top-level dict.update let a patch like ``{access: {teams: [...]}}`` + replace the entire stored ``access`` object, wiping ``access.global=true`` + and ``access.orgs`` entries that the decider intentionally protected by + refusing to allow them in the patch (Veria F1: scope tampering). Now + ``access`` is merged subfield-by-subfield, so a non-admin patch carrying + only ``access.teams`` keeps existing ``access.global`` / ``access.orgs`` + intact. The DB write and the in-memory cache sync both call this so the + two stores can't drift. + """ + patch_copy = dict(patch) + patch_access = patch_copy.pop("access", None) + into.update(patch_copy) + if patch_access is None: + return + existing_access = into.get("access") + if isinstance(existing_access, dict) and isinstance(patch_access, dict): + existing_access.update(patch_access) + else: + into["access"] = patch_access + + +async def _authorize_credential_patch( + *, + credential_name: str, + patch: UpdateCredentialItem, + existing: Optional[CredentialItem], + user_api_key_dict: UserAPIKeyAuth, + prisma_client: "Optional[PrismaClient]", +) -> None: + """Raise 403 unless the caller is allowed to apply ``patch`` to ``existing``. + + The decider widening only applies when the STORED credential is a logging + destination -- a patch body alone can't promote a provider credential into + the decider's allowed paths (Cursor BugBot bypass: ``is_admin_gated_credential_info`` + returned True for any patch carrying ``access``, so a team-admin could PATCH + ``access.teams`` onto a provider credential and reach the decider). + """ + existing_is_logging_gated = existing is not None and is_admin_gated_credential_info(existing.credential_info) + if not existing_is_logging_gated: + _require_proxy_admin(user_api_key_dict) + return + + is_admin = _is_proxy_admin(user_api_key_dict) + team_admin_ids = frozenset() if is_admin else await _caller_grantable_team_ids(user_api_key_dict, prisma_client) + try: + patch_info_typed = ( + CredentialInfo.model_validate(patch.credential_info) if patch.credential_info is not None else None + ) + except ValidationError as ve: + raise HTTPException(status_code=400, detail={"error": _summarize_validation_error(ve)}) + assert existing is not None # narrowed by existing_is_logging_gated + existing_info_typed = CredentialInfo.model_validate(existing.credential_info) + decision = decide_credential_patch( + is_proxy_admin=is_admin, + caller_team_admin_ids=team_admin_ids, + existing_info=existing_info_typed, + patch_info=patch_info_typed, + patch_values=patch.credential_values, + patch_name_changed=(patch.credential_name is not None and patch.credential_name != credential_name), + ) + if isinstance(decision, Deny): + reason = decision.reason if decision.from_user_input else OPAQUE_DENY_REASON + raise HTTPException(status_code=403, detail={"error": reason}) + assert isinstance(decision, Allow) + + +def _patch_to_credential_item(patch: UpdateCredentialItem, credential_name: str) -> CredentialItem: + """Translate the partial PATCH body into the legacy CredentialItem shape + the downstream merge expects (non-None dicts).""" + return CredentialItem( + credential_name=patch.credential_name or credential_name, + credential_values=patch.credential_values or {}, + credential_info=patch.credential_info or {}, + ) + + @router.patch( "/credentials/{credential_name:path}", dependencies=[Depends(user_api_key_auth)], @@ -292,15 +554,30 @@ def update_db_credential( async def update_credential( request: Request, fastapi_response: Response, - credential: CredentialItem, + credential: UpdateCredentialItem, credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ [BETA] endpoint. This might change unexpectedly. + + Both ``credential_values`` and ``credential_info`` are optional; a team-admin + typically patches only ``credential_info.access`` to grant or revoke their + own team. A proxy admin may patch either or both. See + ``decide_credential_patch`` for the exact contract. """ from litellm.proxy.proxy_server import prisma_client + existing = await _credential_for_admin_gate(credential_name, prisma_client) + await _authorize_credential_patch( + credential_name=credential_name, + patch=credential, + existing=existing, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + ) + validate_credential_access(credential.credential_info) + try: if prisma_client is None: raise HTTPException( @@ -311,7 +588,7 @@ async def update_credential( db_credential = await credentials_repository.find_by_name(credential_name) if db_credential is None: raise HTTPException(status_code=404, detail="Credential not found in DB.") - merged_credential = update_db_credential(db_credential, credential) + merged_credential = update_db_credential(db_credential, _patch_to_credential_item(credential, credential_name)) credential_object_jsonified = jsonify_object(merged_credential.model_dump()) await credentials_repository.update_by_name( credential_name, @@ -320,32 +597,49 @@ async def update_credential( "updated_by": user_api_key_dict.user_id, }, ) - - # Sync in-memory credential_list (skip if not in memory - e.g., proxy restarted) - new_name = merged_credential.credential_name - existing_in_memory: Optional[CredentialItem] = None - for cred in litellm.credential_list: - if cred.credential_name == credential_name: - existing_in_memory = cred - break - - if existing_in_memory is not None: - in_memory_values = dict(existing_in_memory.credential_values or {}) - if credential.credential_values: - in_memory_values.update(credential.credential_values) - in_memory_info = dict(existing_in_memory.credential_info or {}) - if credential.credential_info: - in_memory_info.update(credential.credential_info) - updated_in_memory = CredentialItem( - credential_name=new_name, - credential_values=in_memory_values, - credential_info=in_memory_info, - ) - # Remove old entry if renamed, then use upsert_credentials to handle duplicates - if new_name != credential_name: - litellm.credential_list = [c for c in litellm.credential_list if c.credential_name != credential_name] - CredentialAccessor.upsert_credentials([updated_in_memory]) - + _sync_in_memory_credential( + old_name=credential_name, + merged=merged_credential, + patch=credential, + ) return {"success": True, "message": "Credential updated successfully"} except Exception as e: return handle_exception_on_proxy(e) + + +def _sync_in_memory_credential( + *, + old_name: str, + merged: CredentialItem, + patch: UpdateCredentialItem, +) -> None: + """Mirror the DB write into ``litellm.credential_list``. + + Skips when the credential isn't resident in memory (e.g. created on + another scaled instance, restored from DB on the next reload). The + in-memory ``credential_info`` is merged subfield-by-subfield via + ``_merge_credential_info`` so a partial patch can't clobber stored + ``access`` subfields it didn't touch. + """ + existing_in_memory: Optional[CredentialItem] = None + for cred in litellm.credential_list: + if cred.credential_name == old_name: + existing_in_memory = cred + break + if existing_in_memory is None: + return + + in_memory_values = dict(existing_in_memory.credential_values or {}) + if patch.credential_values: + in_memory_values.update(patch.credential_values) + in_memory_info = dict(existing_in_memory.credential_info or {}) + if patch.credential_info: + _merge_credential_info(in_memory_info, patch.credential_info) + updated_in_memory = CredentialItem( + credential_name=merged.credential_name, + credential_values=in_memory_values, + credential_info=in_memory_info, + ) + if merged.credential_name != old_name: + litellm.credential_list = [c for c in litellm.credential_list if c.credential_name != old_name] + CredentialAccessor.upsert_credentials([updated_in_memory]) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0ffb0337545..616995aef8b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -81,6 +81,8 @@ _ENABLE_TEAM_STALE_ALIAS_BYPASS: Optional[bool] = None if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination + from litellm.models.credentials import CredentialItem from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig from litellm.types.proxy.policy_engine import PolicyMatchContext @@ -510,6 +512,235 @@ class KeyAndTeamLoggingSettings: return None +async def _effective_org_id(user_api_key_dict: UserAPIKeyAuth) -> Optional[str]: + """The org this request belongs to, falling back to the team's org when the token + carries none. Team keys frequently have no ``org_id`` on the token, so without this + an org-scoped destination would be invisible at request time even though the write + gate (which loads the team) accepted it. Mirrors the fallback in ``_check_org_budget``. + """ + if user_api_key_dict.org_id is not None: + return user_api_key_dict.org_id + team_id = user_api_key_dict.team_id + if team_id is None: + return None + from litellm.proxy import proxy_server + from litellm.proxy.auth.auth_checks import get_team_object + + if proxy_server.prisma_client is None: + return None + try: + team_obj = await get_team_object( + team_id=team_id, + prisma_client=proxy_server.prisma_client, + user_api_key_cache=proxy_server.user_api_key_cache, + parent_otel_span=getattr(user_api_key_dict, "parent_otel_span", None), + check_db_only=True, + ) + except HTTPException: + return None + return getattr(team_obj, "organization_id", None) + + +async def _union_logging_exporter_names(user_api_key_dict: UserAPIKeyAuth, org_id: Optional[str]) -> set: + """The union of admin-assigned exporter names across the request's identity chain. + + Resolves each level from its OWN record: the key's ``metadata`` is shadowed by the + team's on the auth object, so it is fetched fresh via ``get_key_object``; the org's + metadata is fetched via ``get_org_object`` using the effective ``org_id`` (token org + or team fallback); the team's is already its own on ``team_metadata``. Internal-user + is intentionally not a routing dimension. The lists are admin-owned; the request + never supplies them. Degrades to team-only when no DB is connected (SDK mode). + """ + from litellm.proxy import proxy_server + from litellm.proxy.auth.auth_checks import get_key_object, get_org_object + + names: set = set() + + def _add(metadata: Any) -> None: + if not isinstance(metadata, dict): + return + assigned = metadata.get("logging_exporters") + if isinstance(assigned, list): + names.update(str(name) for name in assigned) + + prisma_client = proxy_server.prisma_client + cache = proxy_server.user_api_key_cache + span = getattr(user_api_key_dict, "parent_otel_span", None) + + # KEY: the key's own metadata (the auth object's .metadata is the team's shadow). + if user_api_key_dict.token and prisma_client is not None: + try: + key_obj = await get_key_object( + hashed_token=user_api_key_dict.token, + prisma_client=prisma_client, + user_api_key_cache=cache, + parent_otel_span=span, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + _add(key_obj.metadata) + except Exception: # noqa: BLE001 + pass + + # TEAM: team_metadata is already the team's own. + _add(user_api_key_dict.team_metadata) + + # ORG: the org's own metadata (central catch-all). + if org_id and prisma_client is not None: + try: + org_obj = await get_org_object( + org_id=org_id, + prisma_client=prisma_client, + user_api_key_cache=cache, + parent_otel_span=span, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + _add(getattr(org_obj, "metadata", None)) + except Exception: # noqa: BLE001 + pass + + return names + + +async def _resolve_logging_exporters( + user_api_key_dict: UserAPIKeyAuth, +) -> "tuple[list, list]": + """Resolve the destinations this request fans out to, as (destinations, backends). + + ``credential_info.access`` is visibility, not enablement: a granted destination + does NOT fire just because the caller can see it. A destination is selected only + when it is an explicit global/default (``auto_enable``) OR it is named in the + identity chain's ``logging_exporters`` (key + team + org) AND its ``access`` grants + the caller. The visibility re-check is defensive: a name that points at a + destination no longer visible to this identity is ignored, so a stale or + cross-tenant assignment can never route traffic out. Each survivor is built via + ``build_destination`` and deduped on (endpoint, headers, resource attributes). + Returns ([], []) when nothing is selected (default-deny). + """ + from litellm.integrations.otel.presets.destinations import build_destination + from litellm.proxy.management_endpoints.logging_exporter_access import ( + access_grants, + is_auto_enable, + ) + + team_id = user_api_key_dict.team_id + org_id = await _effective_org_id(user_api_key_dict) + names = await _union_logging_exporter_names(user_api_key_dict, org_id) + + def _selected(credential: "CredentialItem") -> bool: + info = credential.credential_info or {} + if info.get("credential_type") != "logging": + return False + if is_auto_enable(info): + return True + if credential.credential_name not in names: + return False + return access_grants(info.get("access"), team_id, org_id) + + def _build( + credential: "CredentialItem", + ) -> "Optional[tuple[str, OtelDestination]]": + backend = (credential.credential_info or {}).get("description") + if not backend: + return None + values = {str(key): str(value) for key, value in (credential.credential_values or {}).items()} + destination = build_destination(backend, values) + return None if destination is None else (backend, destination) + + built = tuple( + result + for credential in litellm.credential_list + if _selected(credential) + if (result := _build(credential)) is not None + ) + deduped = { + ( + destination.endpoint, + tuple(sorted(destination.headers.items())), + tuple(sorted(destination.resource_attributes.items())), + ): ( + backend, + destination, + ) + for backend, destination in built + } + destinations = [ + { + "callback_name": backend, + "endpoint": destination.endpoint, + "headers": destination.headers, + "resource_attributes": destination.resource_attributes, + } + for backend, destination in deduped.values() + ] + backends = list(dict.fromkeys(backend for backend, _ in deduped.values())) + return destinations, backends + + +def _request_destination_from_raw(item: object) -> "OtelDestination | None": + from litellm.integrations.otel.model.destination import OtelDestination + + if isinstance(item, OtelDestination): + return item + if not isinstance(item, dict) or not item.get("endpoint"): + return None + try: + return OtelDestination.model_validate(item) + except PydanticValidationError: + return None + + +def _set_request_otel_destinations(destinations: list) -> None: + from litellm.integrations.otel.plumbing.context import set_request_destinations + + set_request_destinations( + tuple(destination for item in destinations if (destination := _request_destination_from_raw(item)) is not None) + ) + + +async def _apply_admin_logging_exporters( + data: dict, + user_api_key_dict: UserAPIKeyAuth, + cached_destinations: "list | None" = None, +) -> None: + """Stamp the resolved fan-out destinations onto ``data`` and activate their + backends. + + The destinations live under ``data["litellm_metadata"]`` (in + ``all_litellm_params``, so scrubbed from the provider request body), not a + top-level key, so an unknown field cannot leak to the provider. Default-deny + means an identity with no assignment gets no per-tenant destination here. + + ``cached_destinations`` -- when ``user_api_key_auth`` already resolved the + destinations on this request (the FastAPI path), reuse the result instead of + running the resolver a second time. The SDK path passes ``None`` and the + resolver runs here. + """ + if cached_destinations is not None: + destinations = list(cached_destinations) + backends = list( + dict.fromkeys( + str(d["callback_name"]) for d in destinations if isinstance(d, dict) and d.get("callback_name") + ) + ) + else: + destinations, backends = await _resolve_logging_exporters(user_api_key_dict) + if not destinations: + return + _set_request_otel_destinations(destinations) + proxy_metadata = data.get("litellm_metadata") + if not isinstance(proxy_metadata, dict): + proxy_metadata = {} + proxy_metadata["otel_destinations"] = destinations + data["litellm_metadata"] = proxy_metadata + # Register on both success and failure: an admin-owned destination must + # capture a failed upstream call (its error gen-AI span) as well as a + # successful one, otherwise a 401/timeout lands a trace with no LLM-call span. + existing_success = data.get("success_callback") or [] + data["success_callback"] = list(dict.fromkeys([*existing_success, *backends])) + existing_failure = data.get("failure_callback") or [] + data["failure_callback"] = list(dict.fromkeys([*existing_failure, *backends])) + + def _get_dynamic_logging_metadata( user_api_key_dict: UserAPIKeyAuth, proxy_config: ProxyConfig ) -> Optional[TeamCallbackMetadata]: @@ -1649,6 +1880,14 @@ async def add_litellm_data_to_request( ) # Team Callbacks controls + # A client must never set or override OTEL destinations; they are admin-owned and + # resolved server-side below. Drop any value carried in the request at either the + # top level OR inside litellm_metadata (the resolver stashes admin-resolved values + # in litellm_metadata.otel_destinations; we wipe the client's first so injection + # via either spot is inert). + data.pop("otel_destinations", None) + if isinstance(data.get("litellm_metadata"), dict): + data["litellm_metadata"].pop("otel_destinations", None) callback_settings_obj = _get_dynamic_logging_metadata( user_api_key_dict=user_api_key_dict, proxy_config=proxy_config ) @@ -1657,10 +1896,17 @@ async def add_litellm_data_to_request( data["failure_callback"] = callback_settings_obj.failure_callback if callback_settings_obj.callback_vars is not None: - # unpack callback_vars in data for k, v in callback_settings_obj.callback_vars.items(): data[k] = v + # Admin-owned exporter assignment: resolve the union of exporters assigned across + # the request's identity chain (key + team + org) into fan-out destinations and + # activate their backends. Default-deny: an unassigned identity gets none. Reuse + # the result ``user_api_key_auth`` already cached on ``request.state`` so the + # resolver runs once per request, not twice. + cached = getattr(getattr(request, "state", None), "otel_destinations", None) + await _apply_admin_logging_exporters(data, user_api_key_dict, cached_destinations=cached) + # Add disabled callbacks from key metadata if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata: disabled_callbacks = user_api_key_dict.metadata["litellm_disabled_callbacks"] diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 8162babef40..d8ff82c6b6e 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -111,15 +111,13 @@ def _is_user_team_admin(user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_Tea return False -async def _is_user_org_admin_for_team(user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_TeamTable) -> bool: - """ - Check if user is an org admin for the team's organization. +async def _is_user_org_admin_for_org_id(user_api_key_dict: UserAPIKeyAuth, organization_id: Optional[str]) -> bool: + """Check if the caller has the ORG_ADMIN role in the given organization. - Returns True if: - - The team belongs to an organization, AND - - The user has org_admin role in that organization + Returns False when ``organization_id`` is falsy or the caller has no user_id, + so the caller can pass an optional org_id directly without branching. """ - if not team_obj.organization_id or not user_api_key_dict.user_id: + if not organization_id or not user_api_key_dict.user_id: return False from litellm.proxy.auth.auth_checks import get_user_object @@ -139,11 +137,18 @@ async def _is_user_org_admin_for_team(user_api_key_dict: UserAPIKeyAuth, team_ob if caller_user is None: return False - for m in caller_user.organization_memberships or []: - if m.organization_id == team_obj.organization_id and m.user_role == LitellmUserRoles.ORG_ADMIN.value: - return True + return any( + m.organization_id == organization_id and m.user_role == LitellmUserRoles.ORG_ADMIN.value + for m in (caller_user.organization_memberships or []) + ) - return False + +async def _is_user_org_admin_for_team(user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_TeamTable) -> bool: + """Check if user is an org admin for the team's organization.""" + return await _is_user_org_admin_for_org_id( + user_api_key_dict=user_api_key_dict, + organization_id=team_obj.organization_id, + ) def _team_member_has_permission( diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2f5c38b0131..4b84b3493e1 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1400,6 +1400,14 @@ async def generate_key_fn( """ try: from litellm.proxy._types import CommonProxyErrors + from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_team, + _is_user_team_admin, + ) + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + LOGGING_EXPORTERS_KEY, + validate_logging_exporter_assignment, + ) from litellm.proxy.proxy_server import ( prisma_client, user_api_key_cache, @@ -1487,6 +1495,27 @@ async def generate_key_fn( route=KeyManagementRoutes.KEY_GENERATE, ) + # Team-admin of the key's team or org-admin of that team's org may + # write metadata.logging_exporters on team-owned keys. Personal keys + # (no team_table) stay proxy-admin only. Skip the role lookup when + # the field isn't in the payload to keep /key/generate cheap for the + # common case. + if isinstance(data.metadata, dict) and LOGGING_EXPORTERS_KEY in data.metadata: + validate_logging_exporter_assignment( + data.metadata, + user_api_key_dict, + caller_is_team_admin=( + team_table is not None + and _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_table) + ), + caller_is_org_admin=( + team_table is not None + and await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team_table) + ), + scope_team_id=getattr(team_table, "team_id", None), + scope_org_id=getattr(team_table, "organization_id", None), + ) + if team_table is not None: await _check_team_key_limits( team_table=team_table, @@ -1662,6 +1691,34 @@ async def generate_service_account_key_fn( route=KeyManagementRoutes.KEY_GENERATE_SERVICE_ACCOUNT, ) + # Same logging_exporters gate as /key/generate. Without this, a caller + # eligible for service-account creation could set metadata.logging_exporters + # and route future traces to a destination they aren't allowed to assign + # (Veria F3). Skip the lookup unless the field is being written. + from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_team, + _is_user_team_admin, + ) + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + LOGGING_EXPORTERS_KEY, + validate_logging_exporter_assignment, + ) + + if isinstance(data.metadata, dict) and LOGGING_EXPORTERS_KEY in data.metadata: + validate_logging_exporter_assignment( + data.metadata, + user_api_key_dict, + caller_is_team_admin=( + team_table is not None and _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_table) + ), + caller_is_org_admin=( + team_table is not None + and await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team_table) + ), + scope_team_id=getattr(team_table, "team_id", None), + scope_org_id=getattr(team_table, "organization_id", None), + ) + data.user_id = None # do not allow user_id to be set for service account keys return await _common_key_generation_helper( @@ -2318,7 +2375,7 @@ async def _validate_update_key_data( @router.post("/key/update", tags=["key management"], dependencies=[Depends(user_api_key_auth)]) @management_endpoint_wrapper -async def update_key_fn( +async def update_key_fn( # noqa: C901 request: Request, data: UpdateKeyRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -2395,6 +2452,13 @@ async def update_key_fn( }' ``` """ + from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_team, + _is_user_team_admin, + ) + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + validate_logging_exporter_assignment, + ) from litellm.proxy.proxy_server import ( llm_router, premium_user, @@ -2421,6 +2485,43 @@ async def update_key_fn( prisma_client=prisma_client, ) + # logging-exporters validation runs once the key's team is known so + # a team-admin or org-admin of that team can attach destinations. + # The validator no-ops when the effective value doesn't change; pass + # the stored metadata so removal-via-omission gates too (Veria F4). + if isinstance(data.metadata, dict): + _existing_key_metadata = ( + existing_key_row.metadata if isinstance(getattr(existing_key_row, "metadata", None), dict) else None + ) + _key_team_id = getattr(existing_key_row, "team_id", None) + _key_team = None + if _key_team_id is not None: + try: + _key_team = await get_team_object( + team_id=_key_team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_dict.parent_otel_span, + check_db_only=True, + ) + except HTTPException: + _key_team = None + validate_logging_exporter_assignment( + data.metadata, + user_api_key_dict, + caller_is_team_admin=( + _key_team is not None + and _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=_key_team) + ), + caller_is_org_admin=( + _key_team is not None + and await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=_key_team) + ), + existing_metadata=_existing_key_metadata, + scope_team_id=getattr(_key_team, "team_id", None), + scope_org_id=getattr(_key_team, "organization_id", None), + ) + await _validate_update_key_data( data=data, existing_key_row=existing_key_row, @@ -4272,7 +4373,7 @@ async def _execute_virtual_key_regeneration( dependencies=[Depends(user_api_key_auth)], ) @management_endpoint_wrapper -async def regenerate_key_fn( +async def regenerate_key_fn( # noqa: C901 key: Optional[str] = None, data: Optional[RegenerateKeyRequest] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -4460,21 +4561,65 @@ async def regenerate_key_fn( # Gate access_group_ids on regenerate, same as /key/generate and # /key/update. Use the existing key's team since the body may omit it. - if data is not None and data.access_group_ids: - regenerate_team_table: Optional[LiteLLM_TeamTableCachedObj] = None - if _key_in_db.team_id is not None: + regenerate_team_table: Optional[LiteLLM_TeamTableCachedObj] = None + if _key_in_db.team_id is not None: + try: regenerate_team_table = await get_team_object( team_id=_key_in_db.team_id, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, check_db_only=True, ) + except HTTPException: + regenerate_team_table = None + if data is not None and data.access_group_ids: TeamMemberPermissionChecks.enforce_member_can_assign_access_groups( user_api_key_dict=user_api_key_dict, team_table=regenerate_team_table, access_group_ids=data.access_group_ids, ) + # logging_exporters gate on regenerate matches /key/generate and + # /key/update. Without this, a key owner could set + # metadata.logging_exporters on /key/{id}/regenerate and route + # future traces to a destination they aren't allowed to assign + # (Veria F3). The validator no-ops when the effective value + # doesn't change; pass stored metadata so removal-via-omission + # gates too (Veria F4). + if data is not None and isinstance(data.metadata, dict): + from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_team, + _is_user_team_admin, + ) + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + validate_logging_exporter_assignment, + ) + + _regen_existing_metadata = ( + _key_in_db.metadata if isinstance(getattr(_key_in_db, "metadata", None), dict) else None + ) + validate_logging_exporter_assignment( + data.metadata, + user_api_key_dict, + caller_is_team_admin=( + regenerate_team_table is not None + and _is_user_team_admin( + user_api_key_dict=user_api_key_dict, + team_obj=regenerate_team_table, + ) + ), + caller_is_org_admin=( + regenerate_team_table is not None + and await _is_user_org_admin_for_team( + user_api_key_dict=user_api_key_dict, + team_obj=regenerate_team_table, + ) + ), + existing_metadata=_regen_existing_metadata, + scope_team_id=getattr(regenerate_team_table, "team_id", None), + scope_org_id=getattr(regenerate_team_table, "organization_id", None), + ) + verbose_proxy_logger.info( "Key regeneration requested: key_alias=%s", getattr(_key_in_db, "key_alias", None), diff --git a/litellm/proxy/management_endpoints/logging_exporter_access.py b/litellm/proxy/management_endpoints/logging_exporter_access.py new file mode 100644 index 00000000000..a3d4242838c --- /dev/null +++ b/litellm/proxy/management_endpoints/logging_exporter_access.py @@ -0,0 +1,41 @@ +"""Shared access predicate for admin-owned logging destinations. + +``credential_info.access`` answers "who may see/assign this destination" — it is +visibility, decoupled from enablement (which lives in ``metadata.logging_exporters`` +and the explicit ``auto_enable`` flag). The request-time resolver and the write-time +validator gate on the SAME predicate so "visible" means the same thing on both sides: +a team/org admin can only assign destinations they can see, and the resolver +defensively re-checks visibility at request time. +""" + +from typing import Optional + +AUTO_ENABLE_KEY = "auto_enable" + + +def access_grants(access: object, team_id: Optional[str], org_id: Optional[str]) -> bool: + """Whether a destination's ``access`` makes it visible to this identity. + + ``global`` reaches everyone; otherwise the identity's team or org must be listed. + A missing or malformed ``access`` grants no one (fail closed): visibility must be + an explicit admin grant, never an accident of an absent field. + """ + if not isinstance(access, dict): + return False + if access.get("global") is True: + return True + teams = access.get("teams") + if team_id is not None and isinstance(teams, (list, tuple)) and team_id in teams: + return True + orgs = access.get("orgs") + return org_id is not None and isinstance(orgs, (list, tuple)) and org_id in orgs + + +def is_auto_enable(credential_info: object) -> bool: + """Whether a destination is an explicit global/default (auto-enabled everywhere). + + This is the deliberate replacement for the old behavior where ``access.global`` + implicitly auto-enabled a destination for every request. Enablement is now opt-in: + only ``auto_enable`` turns a destination on without being named. + """ + return isinstance(credential_info, dict) and credential_info.get(AUTO_ENABLE_KEY) is True diff --git a/litellm/proxy/management_endpoints/logging_exporter_validation.py b/litellm/proxy/management_endpoints/logging_exporter_validation.py new file mode 100644 index 00000000000..a3f268f7e74 --- /dev/null +++ b/litellm/proxy/management_endpoints/logging_exporter_validation.py @@ -0,0 +1,220 @@ +"""Validation for admin-owned logging-exporter assignment on key/team/org. + +An identity's ``metadata.logging_exporters`` binds it to admin-owned trace +destinations. Every name must be a registered logging credential, the caller must +hold a role that authorizes the write (proxy admin always; team admin and org admin +in specific contexts), AND a non-proxy-admin may only name destinations whose +``credential_info.access`` makes them visible to the scope being written (the key's +team, or the team/org being updated). Visibility and enablement are separate: a +destination granted to a team is assignable by that team's admin, but assigning it is +what enables it. The resolver (``litellm_pre_call_utils``) re-checks visibility at +request time, so this gate and the resolver agree on what "visible" means. +""" + +from typing import Optional + +from fastapi import HTTPException, status + +import litellm +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.management_endpoints.logging_exporter_access import ( + access_grants, + is_auto_enable, +) + +LOGGING_EXPORTERS_KEY = "logging_exporters" + + +def is_admin_gated_credential_info(credential_info: Optional[dict]) -> bool: + """Whether a credential write must be proxy-admin only. + + True when the credential is a logging destination or carries an ``access`` grant, + since both control where other tenants' traces are exported. + """ + if not isinstance(credential_info, dict): + return False + return credential_info.get("credential_type") == "logging" or "access" in credential_info + + +def validate_credential_access(credential_info: Optional[dict]) -> None: + """Validate ``credential_info.access`` shape when the write sets one. + + No-op when ``access`` is absent. Otherwise it must be an object whose ``global`` (if + present) is a bool and whose ``teams``/``orgs`` (if present) are lists of strings. + Per-key access is intentionally unsupported on a destination. + """ + if not isinstance(credential_info, dict) or "access" not in credential_info: + return + access = credential_info["access"] + if not isinstance(access, dict): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "credential_info.access must be an object"}, + ) + if "global" in access and not isinstance(access["global"], bool): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "access.global must be a boolean"}, + ) + for field in ("teams", "orgs"): + bucket = access.get(field) + if bucket is not None and not (isinstance(bucket, list) and all(isinstance(item, str) for item in bucket)): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": f"access.{field} must be a list of strings"}, + ) + + +def _logging_credentials_by_name() -> dict[str, dict]: + return { + credential.credential_name: (credential.credential_info or {}) + for credential in litellm.credential_list + if (credential.credential_info or {}).get("credential_type") == "logging" + } + + +def _logging_credential_names() -> set[str]: + return set(_logging_credentials_by_name()) + + +def _reject_unassignable_destinations( + exporters: list[str], + *, + scope_team_id: Optional[str], + scope_org_id: Optional[str], +) -> None: + """Reject names a non-proxy-admin cannot assign in this scope. + + A destination is assignable when it is an explicit global/default + (``auto_enable``) or its ``access`` grants the scope being written (the key's + team, or the team/org being updated). Names are already known logging + credentials by the time this runs, so a missing entry means a benign race; we + fail closed on it. + """ + by_name = _logging_credentials_by_name() + unassignable = [ + name + for name in exporters + if not ( + is_auto_enable(by_name.get(name)) + or access_grants((by_name.get(name) or {}).get("access"), scope_team_id, scope_org_id) + ) + ] + if unassignable: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": ( + "You can only assign logging destinations granted to your team " + f"or organization. Not granted: {unassignable}" + ) + }, + ) + + +def _validate_exporters_shape_and_names(exporters: object) -> None: + """Common shape + registry check shared by every entry point.""" + if not isinstance(exporters, list): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "logging_exporters must be a list of credential names"}, + ) + known = _logging_credential_names() + unknown = [name for name in exporters if name not in known] + if unknown: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": ( + f"Unknown or non-logging credential(s): {unknown}. Register them " + "as logging credentials before assigning." + ) + }, + ) + + +def _exporter_value_changes( + requested_metadata: Optional[dict], + existing_metadata: Optional[dict], +) -> bool: + """True if the effective ``metadata.logging_exporters`` value would change. + + An update endpoint that REPLACES stored metadata with ``requested_metadata`` + will drop ``logging_exporters`` when the new payload omits it. So a write + requires authorization whenever: + + - the new metadata sets ``logging_exporters`` (the previously-handled case), OR + - the new metadata is provided but omits ``logging_exporters`` while the + stored metadata had one (removal-via-omission, Veria F4). + + Returns False when stored and requested values match exactly, or when the + update doesn't touch metadata at all. + """ + if not isinstance(requested_metadata, dict): + return False + new_has = LOGGING_EXPORTERS_KEY in requested_metadata + existing = existing_metadata.get(LOGGING_EXPORTERS_KEY) if isinstance(existing_metadata, dict) else None + existing_has = existing is not None + if not new_has and not existing_has: + return False + if new_has and not existing_has: + return True + if not new_has and existing_has: + return True + return requested_metadata.get(LOGGING_EXPORTERS_KEY) != existing + + +def validate_logging_exporter_assignment( + metadata: Optional[dict], + user_api_key_dict: UserAPIKeyAuth, + *, + caller_is_team_admin: bool = False, + caller_is_org_admin: bool = False, + existing_metadata: Optional[dict] = None, + scope_team_id: Optional[str] = None, + scope_org_id: Optional[str] = None, +) -> None: + """Validate a ``metadata.logging_exporters`` write on key / team / org endpoints. + + No-op when the update does not change the effective ``logging_exporters`` + value. Proxy admins always pass. Caller-provided flags widen the allow-list + per endpoint: + + - ``/team/update``: pass ``caller_is_org_admin`` from the loaded team's org. + - ``/key/generate``/``/key/update``: pass both flags from the key's team. + - ``/team/new``/``/organization/*``: pass neither (proxy-admin only). + + ``scope_team_id``/``scope_org_id`` are the team and org the write lands in (the + key's team, or the team/org being updated). A non-proxy-admin may only name + destinations visible to that scope: this is what stops a team admin from routing a + key's traces to a destination scoped to a different team. Proxy admins skip the + scope check; they can assign anything, but the resolver still only fires a named + destination for identities it is visible to. + + Update paths replace stored metadata wholesale, so a caller can drop an + admin-assigned exporter by sending ``metadata`` without + ``logging_exporters``. Pass ``existing_metadata`` from the loaded row so + removal-via-omission is gated too (Veria F4). On create paths the existing + value is implicitly ``None`` and the validator behaves as before. + + Every exporter name (when present) must resolve to a registered logging credential. + """ + if not _exporter_value_changes(metadata, existing_metadata): + return + is_proxy_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN + if not (is_proxy_admin or caller_is_team_admin or caller_is_org_admin): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": ( + "Only the proxy admin, a team admin of this team, or an " + "org admin of this team's organization can assign logging " + "exporters" + ) + }, + ) + requested = metadata.get(LOGGING_EXPORTERS_KEY) if isinstance(metadata, dict) else None + if requested is not None: + _validate_exporters_shape_and_names(requested) + if not is_proxy_admin: + _reject_unassignable_destinations(requested, scope_team_id=scope_team_id, scope_org_id=scope_org_id) diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 138a55d9227..178c518c8b2 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -198,6 +198,11 @@ async def new_organization( }' ``` """ + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + validate_logging_exporter_assignment, + ) + + validate_logging_exporter_assignment(getattr(data, "metadata", None), user_api_key_dict) from litellm.proxy.proxy_server import ( litellm_proxy_admin_name, @@ -453,6 +458,12 @@ async def update_organization( # Create validated data model data = LiteLLM_OrganizationTableUpdate(**raw_data_with_flat_budget_fields) + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + validate_logging_exporter_assignment, + ) + + validate_logging_exporter_assignment(getattr(data, "metadata", None), user_api_key_dict) + # Validate budget values are not negative if data.max_budget is not None and (not math.isfinite(data.max_budget) or data.max_budget < 0): raise HTTPException( diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0ea2b9e05f9..463d83c9688 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -988,6 +988,13 @@ async def new_team( ``` """ try: + from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_org_id, + ) + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + LOGGING_EXPORTERS_KEY, + validate_logging_exporter_assignment, + ) from litellm.proxy.management_helpers.audit_logs import ( get_audit_log_changed_by, ) @@ -999,6 +1006,21 @@ async def new_team( user_api_key_cache, ) + # New team has no admins yet, so only proxy admin or an org admin of + # the destination org may assign logging exporters at creation time. + # Skip the org-admin lookup entirely when the field isn't being + # written, to avoid hitting the cache for unrelated /team/new calls. + if isinstance(data.metadata, dict) and LOGGING_EXPORTERS_KEY in data.metadata: + validate_logging_exporter_assignment( + data.metadata, + user_api_key_dict, + caller_is_org_admin=await _is_user_org_admin_for_org_id( + user_api_key_dict=user_api_key_dict, + organization_id=data.organization_id, + ), + scope_org_id=data.organization_id, + ) + if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -1637,6 +1659,12 @@ async def update_team( ``` """ try: + from litellm.proxy.management_endpoints.common_utils import ( + _is_user_org_admin_for_team, + ) + from litellm.proxy.management_endpoints.logging_exporter_validation import ( + validate_logging_exporter_assignment, + ) from litellm.proxy.proxy_server import ( litellm_proxy_admin_name, llm_router, @@ -1685,11 +1713,34 @@ async def update_team( ) # Verify caller has access to manage this team + team_for_auth = LiteLLM_TeamTable(**existing_team_row.model_dump()) await _verify_team_access( - team_obj=LiteLLM_TeamTable(**existing_team_row.model_dump()), + team_obj=team_for_auth, user_api_key_dict=user_api_key_dict, ) + # logging_exporters on /team/update is proxy-admin or org-admin only: + # team-admins are blocked at the route gate (test_team_update_authz_ + # matrix pins this) and the role matrix documents ❌ for team-admin on + # this path. Pass only the org-admin flag so the validator can't + # silently grant team-admins if the route gate is ever widened. The + # validator no-ops when the effective value doesn't change; pass the + # stored metadata so removal-via-omission gates too (Veria F4). + if isinstance(data.metadata, dict): + existing_team_metadata = ( + existing_team_row.metadata if isinstance(existing_team_row.metadata, dict) else None + ) + validate_logging_exporter_assignment( + data.metadata, + user_api_key_dict, + caller_is_org_admin=await _is_user_org_admin_for_team( + user_api_key_dict=user_api_key_dict, team_obj=team_for_auth + ), + existing_metadata=existing_team_metadata, + scope_team_id=getattr(team_for_auth, "team_id", None), + scope_org_id=getattr(team_for_auth, "organization_id", None), + ) + _check_passthrough_routes_caller_permission(data, user_api_key_dict, entity="team") if data.soft_budget is not None: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 279e9b15fe7..dda7171d961 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2923,6 +2923,20 @@ OPENAI_RESPONSE_HEADERS = [ ] +class OtelDestinationParams(TypedDict, total=False): + """A resolved, admin-owned OTLP destination carried server-side only. + + Populated by the proxy from the exporters assigned to a request's identity + chain; never read from a request body or metadata. The v2 logger validates and + exports through it. ``callback_name`` is the OTEL backend this destination + belongs to, so fan-out routes each destination to the right backend's logger. + """ + + callback_name: str + endpoint: str + headers: Dict[str, str] + + class StandardCallbackDynamicParams(TypedDict, total=False): # Langfuse dynamic params langfuse_public_key: Optional[str] @@ -2970,6 +2984,12 @@ class StandardCallbackDynamicParams(TypedDict, total=False): turn_off_message_logging: Optional[bool] # when true will not log messages litellm_disabled_callbacks: Optional[List[str]] + # Admin-owned OTEL v2 destinations, resolved server-side from the exporters + # assigned to the request's identity chain (key/team/user/org), fanned out to. + # Never request-settable: absent from the request-read whitelist in + # initialize_dynamic_callback_params, so a request body/metadata cannot set it. + otel_destinations: Optional[List[OtelDestinationParams]] + class CustomPricingLiteLLMParams(BaseModel): ## CUSTOM PRICING ## @@ -3567,6 +3587,9 @@ from litellm.models.credentials import CredentialItem as CredentialItem # noqa: from litellm.models.credentials import ( # noqa: E402 CreateCredentialItem as CreateCredentialItem, ) +from litellm.models.credentials import ( # noqa: E402 + UpdateCredentialItem as UpdateCredentialItem, +) class ExtractedFileData(TypedDict): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 19eef284b91..dd9a027f870 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -465,6 +465,16 @@ def test_otlp_traces_endpoint_normalization(): norm("https://x.splunk.com/v2/trace/otlp") == "https://x.splunk.com/v2/trace/otlp" ) + # Langtrace ingests at /api/trace (a complete path, not an OTLP base) — appending + # /v1/traces 404s, so it must be left intact (regression for the double-append bug). + assert ( + norm("https://app.langtrace.ai/api/trace") + == "https://app.langtrace.ai/api/trace" + ) + assert ( + norm("https://app.langtrace.ai/api/trace/") + == "https://app.langtrace.ai/api/trace" + ) assert norm(None) is None diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py index f7c0b5452fe..253f361bee0 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py @@ -1,88 +1,97 @@ -"""Per-request multi-tenant credential routing (V1 parity).""" +"""Per-tenant tracer routing on admin-owned OTEL destinations, with fan-out. + +A request's identity chain is assigned a set of admin-owned exporters; the v2 logger +fans the trace out to all of them (plus the configured/global exporter), and never +routes on request-supplied vendor credentials. These tests lock the contract: the +request cannot route a trace, each destination's endpoint follows its resolved host +(cross-host fix), the configured exporters are kept (global also receives), and a +logger only exports the destinations tagged with its own backend. +""" import os import sys +import pytest + sys.path.insert(0, os.path.abspath("../../../..")) from opentelemetry.trace import NoOpTracer from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.presets import dynamic_otlp_headers +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.model.metadata import LLMCallEvent from litellm.integrations.otel.plumbing.routing import TenantTracerCache def _cache(callback_name, exporters=None): - cfg = OpenTelemetryV2Config(exporters=exporters or [ExporterSpec(kind="in_memory")]) + cfg = OpenTelemetryV2Config( + exporters=exporters or [ExporterSpec(kind="in_memory", owner=callback_name)] + ) return TenantTracerCache(cfg, callback_name, "litellm") -# --- header builders mirror the V1 construct_dynamic_otel_headers overrides --- # - - -def test_arize_dynamic_headers(): - headers = dynamic_otlp_headers( - "arize", {"arize_space_id": "S", "arize_api_key": "K"} +def _dest(endpoint, auth="Basic AAAA", backend="langfuse_otel"): + return OtelDestination( + endpoint=endpoint, headers={"Authorization": auth}, callback_name=backend ) - assert headers == {"arize-space-id": "S", "api_key": "K"} -def test_arize_space_key_overrides_space_id(): - headers = dynamic_otlp_headers( - "arize", {"arize_space_id": "S", "arize_space_key": "SK"} +def _event(destinations): + return LLMCallEvent.from_dict( + { + "standard_callback_dynamic_params": {"otel_destinations": destinations}, + "call_type": "acompletion", + "model": "gpt-4o", + } ) - assert headers == {"arize-space-id": "SK"} -def test_langfuse_dynamic_headers_need_both_keys(): - assert dynamic_otlp_headers("langfuse_otel", {"langfuse_public_key": "pk"}) is None - headers = dynamic_otlp_headers( - "langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} - ) - assert headers is not None and "Authorization" in headers +# --- routing only happens for admin destinations --------------------------- # -def test_weave_dynamic_headers(): - headers = dynamic_otlp_headers( - "weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"} - ) - assert headers is not None - assert "Authorization" in headers and headers["project_id"] == "p" - - -def test_non_participating_callbacks_have_no_routing(): - # Phoenix subclasses the base in V1 (no override) → no dynamic routing. - assert dynamic_otlp_headers("arize_phoenix", {"arize_api_key": "K"}) is None - assert dynamic_otlp_headers("langtrace", {"arize_api_key": "K"}) is None - assert dynamic_otlp_headers(None, {"arize_api_key": "K"}) is None - - -def test_no_dynamic_params_is_no_routing(): - assert dynamic_otlp_headers("arize", None) is None - assert dynamic_otlp_headers("arize", {}) is None - - -# --- TenantTracerCache routes + caches a TracerProvider per credential set --- # - - -def test_provider_cached_per_credential_set(): - cache = _cache("arize") +def test_no_destinations_uses_default_tracer(): + cache = _cache("langfuse_otel") default = NoOpTracer() - creds_a = {"arize_space_id": "S", "arize_api_key": "K"} - creds_b = {"arize_space_id": "S2", "arize_api_key": "K2"} + assert cache.tracer_for(default, ()) is default + assert cache._providers == {} - cache.tracer_for(default, creds_a) - cache.tracer_for(default, creds_a) # same set → reuse, no new provider + +def test_provider_cached_per_destination_set(): + cache = _cache("langfuse_otel") + default = NoOpTracer() + a = (_dest("https://eu.example/v1", "Basic A"),) + b = (_dest("https://eu.example/v1", "Basic B"),) + + cache.tracer_for(default, a) + cache.tracer_for(default, a) # same set -> reuse assert len(cache._providers) == 1 - cache.tracer_for(default, creds_b) # new set → new provider + cache.tracer_for(default, b) # different creds -> new provider assert len(cache._providers) == 2 +def test_different_host_is_a_distinct_provider(): + """Two destinations with identical headers but different hosts must not collide; + the cache key includes each endpoint.""" + cache = _cache("langfuse_otel") + default = NoOpTracer() + eu = (_dest("https://cloud.langfuse.com/api/public/otel", "Basic X"),) + us = (_dest("https://us.cloud.langfuse.com/api/public/otel", "Basic X"),) + cache.tracer_for(default, eu) + cache.tracer_for(default, us) + assert len(cache._providers) == 2 + + +def test_destination_set_is_order_independent(): + cache = _cache("langfuse_otel") + default = NoOpTracer() + a = _dest("https://a/v1", "Basic A") + b = _dest("https://b/v1", "Basic B") + cache.tracer_for(default, (a, b)) + cache.tracer_for(default, (b, a)) # same set, different order -> one provider + assert len(cache._providers) == 1 + + def test_provider_cache_is_bounded_and_evicts_lru(monkeypatch): - # The cache key derives from request-supplied dynamic credentials, so it - # must be bounded — an unbounded cache lets a caller spawn one provider (and - # its background exporter thread) per unique credential set. On overflow the - # least-recently-used provider is evicted and shut down. from litellm.integrations.otel.plumbing import routing as routing_mod monkeypatch.setattr(routing_mod, "_MAX_CACHED_PROVIDERS", 2) @@ -90,61 +99,49 @@ def test_provider_cache_is_bounded_and_evicts_lru(monkeypatch): monkeypatch.setattr( routing_mod, "_shutdown_provider", lambda p: shut_down.append(p) ) - - cache = _cache("arize") + cache = _cache("langfuse_otel") default = NoOpTracer() - - def creds(space): - return {"arize_space_id": space, "arize_api_key": "K"} - - cache.tracer_for(default, creds("1")) - cache.tracer_for(default, creds("2")) - cache.tracer_for(default, creds("1")) # touch "1" → "2" is now LRU - cache.tracer_for(default, creds("3")) # overflow → evict "2" - + cache.tracer_for(default, (_dest("https://1/v1"),)) + cache.tracer_for(default, (_dest("https://2/v1"),)) + cache.tracer_for(default, (_dest("https://1/v1"),)) # touch "1" -> "2" is LRU + cache.tracer_for(default, (_dest("https://3/v1"),)) # overflow -> evict "2" assert len(cache._providers) == 2 - assert len(shut_down) == 1 # exactly the evicted provider was shut down + assert len(shut_down) == 1 -def test_no_dynamic_params_uses_default_tracer(): - cache = _cache("arize") - default = NoOpTracer() - assert cache.tracer_for(default, {}) is default - assert cache._providers == {} +# --- fan-out: keep the configured exporters, append one per destination ----- # -def test_non_participating_callback_uses_default_tracer(): - cache = _cache("arize_phoenix") - default = NoOpTracer() - assert cache.tracer_for(default, {"arize_api_key": "K"}) is default - assert cache._providers == {} - - -def test_dynamic_headers_applied_to_otlp_exporter_only(): +@pytest.mark.parametrize("owner", ["langfuse_otel", "arize", "weave_otel"]) +def test_fan_out_appends_destination_with_resolved_endpoint(owner): + # The configured (global) exporter is kept; each destination is appended with its + # OWN resolved endpoint + headers (the cross-host fix, per owner). cache = _cache( - "arize", + owner, exporters=[ - ExporterSpec(kind="otlp_http", owner="arize"), - ExporterSpec(kind="in_memory", owner="arize"), + ExporterSpec( + kind="otlp_http", + endpoint="https://env-host.example/v1", + headers="Authorization=Basic ENV", + owner=owner, + ) ], ) - new_cfg = cache._config_with_headers({"arize-space-id": "S", "api_key": "K"}) - otlp, in_mem = new_cfg.exporters - assert otlp.headers == "arize-space-id=S,api_key=K" - assert in_mem.headers is None # console/in_memory left untouched + new = cache._config_with_destinations( + (_dest("https://resolved.example/v1", "Basic TEAM", backend=owner),) + ) + # global kept verbatim + assert new.exporters[0].endpoint == "https://env-host.example/v1" + assert new.exporters[0].headers == "Authorization=Basic ENV" + # destination appended at the resolved host with its own auth + assert new.exporters[-1].endpoint == "https://resolved.example/v1" + assert new.exporters[-1].headers == "Authorization=Basic TEAM" + assert len(new.exporters) == 2 -def test_dynamic_headers_do_not_leak_to_other_owners_exporter(): - """A tenant's Arize credentials must never be stamped onto a co-configured - exporter owned by a different backend (a self-hosted collector, Langfuse). - - Regression for the cross-backend credential leak: ``_config_with_headers`` - used to rewrite the headers of every OTLP exporter, so one request carrying - a team's Arize key clobbered the base collector's and Langfuse's headers - with that key. - """ +def test_fan_out_preserves_co_configured_exporters(): cache = _cache( - "arize", + "langfuse_otel", exporters=[ ExporterSpec( kind="otlp_http", @@ -154,22 +151,352 @@ def test_dynamic_headers_do_not_leak_to_other_owners_exporter(): ), ExporterSpec( kind="otlp_http", - endpoint="https://cloud.langfuse.com/api/public/otel", - headers="Authorization=Basic base-langfuse", + endpoint="https://us.cloud.langfuse.com/api/public/otel", + headers="Authorization=Basic ENV", owner="langfuse_otel", ), - ExporterSpec( - kind="otlp_grpc", - endpoint="https://otlp.arize.com/v1", - headers="space_id=base,api_key=base", - owner="arize", - ), ], ) - new_cfg = cache._config_with_headers( - {"arize-space-id": "TEAMX", "api_key": "TEAMX_KEY"} + new = cache._config_with_destinations( + (_dest("https://cloud.langfuse.com/api/public/otel", "Basic TEAM"),) ) - by_owner = {e.owner: e.headers for e in new_cfg.exporters} - assert by_owner["arize"] == "arize-space-id=TEAMX,api_key=TEAMX_KEY" - assert by_owner[None] == "x=base-collector" - assert by_owner["langfuse_otel"] == "Authorization=Basic base-langfuse" + # both originals preserved unchanged (no rewrite/leak) + assert new.exporters[0].endpoint == "http://self-hosted-collector:4318" + assert new.exporters[0].headers == "x=base-collector" + assert new.exporters[1].headers == "Authorization=Basic ENV" + # exactly one appended + assert new.exporters[-1].endpoint == "https://cloud.langfuse.com/api/public/otel" + assert len(new.exporters) == 3 + + +def test_fan_out_to_many_destinations_is_one_provider_with_all_exporters(): + cache = _cache( + "langfuse_otel", + exporters=[ + ExporterSpec( + kind="otlp_http", endpoint="https://env/v1", owner="langfuse_otel" + ) + ], + ) + new = cache._config_with_destinations( + (_dest("https://a/v1", "Basic A"), _dest("https://b/v1", "Basic B")) + ) + # global + 2 destinations -> 3 span processors, one provider, one span copied to all + assert [e.endpoint for e in new.exporters] == [ + "https://env/v1", + "https://a/v1", + "https://b/v1", + ] + cache.tracer_for(NoOpTracer(), (_dest("https://a/v1"), _dest("https://b/v1"))) + assert len(cache._providers) == 1 + + +# --- gen-AI span Resource must match its fanned-out parents ----------------- # +# +# The gen-AI LLM-call span is emitted through the TenantTracerCache clone +# provider, while the proxy-internal spans (server/auth/db) are forwarded by +# TenantFanOutSpanProcessor, which wraps each with the destination's backend- +# required Resource attrs (Arize needs model_id / arize.project.name). If the +# clone provider does NOT also carry those attrs, the gen-AI span reaches Arize +# with only service.name and Arize renders it as an orphaned subtree. These pin +# that the clone Resource carries the same attrs the fan-out wrap injects. + + +def _dest_with_resource(endpoint, backend, resource_attributes): + return OtelDestination( + endpoint=endpoint, + headers={"Authorization": "Basic AAAA"}, + callback_name=backend, + resource_attributes=resource_attributes, + ) + + +def test_clone_config_carries_destination_resource_attrs(): + cache = _cache("arize") + new = cache._config_with_destinations( + ( + _dest_with_resource( + "https://otlp.arize.com/v1", + "arize", + {"model_id": "team-b-proj", "arize.project.name": "team-b-proj"}, + ), + ) + ) + assert new.resource_attributes["model_id"] == "team-b-proj" + assert new.resource_attributes["arize.project.name"] == "team-b-proj" + + +def test_clone_config_carries_builder_declared_resource_attrs(monkeypatch): + """End-to-end: an arize credential that omits the project gets ARIZE_PROJECT_NAME + folded in at BUILD time (build_destination), and the clone config then carries + those Resource attrs generically -- the gen-AI span lands in the same project as + its fan-out'd parents. The clone path itself is backend-agnostic; it reads + whatever the builder declared.""" + monkeypatch.setenv("ARIZE_PROJECT_NAME", "env-proj") + from litellm.integrations.otel.presets.destinations import build_destination + + dest = build_destination("arize", {"arize_space_id": "s", "arize_api_key": "k"}) + assert dest is not None + cache = _cache("arize") + new = cache._config_with_destinations((dest,)) + assert new.resource_attributes["model_id"] == "env-proj" + assert new.resource_attributes["arize.project.name"] == "env-proj" + + +def test_clone_provider_emits_genai_span_with_destination_resource(): + """End-to-end regression: the span the clone provider actually exports must + carry the destination's Resource attrs. Pre-fix this Resource was + service.name only, orphaning the gen-AI span in Arize.""" + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + + from litellm.integrations.otel.plumbing.providers import ( + build_tracer_provider, + get_tracer, + ) + + cfg = OpenTelemetryV2Config( + service_name="litellm-proxy", + exporters=[ExporterSpec(kind="in_memory", owner="arize")], + ) + cache = TenantTracerCache(cfg, "arize", "litellm") + dest = _dest_with_resource( + "https://otlp.arize.com/v1", + "arize", + {"model_id": "team-b-proj", "arize.project.name": "team-b-proj"}, + ) + tracer = cache.tracer_for(get_tracer(build_tracer_provider(cfg)), (dest,)) + with tracer.start_as_current_span("chat anthropic-haiku") as span: + span.set_attribute("gen_ai.operation.name", "chat") + + provider = next(iter(cache._providers.values())) + provider.force_flush() + captured = [] + for proc in provider._active_span_processor._span_processors: + exporter = getattr(proc, "span_exporter", None) + if isinstance(exporter, InMemorySpanExporter): + captured = exporter.get_finished_spans() + assert captured, "clone provider exported no span to its in-memory exporter" + resource_attrs = dict(captured[0].resource.attributes) + assert resource_attrs.get("model_id") == "team-b-proj" + assert resource_attrs.get("arize.project.name") == "team-b-proj" + + +# --- security: request credentials never route a trace --------------------- # + + +@pytest.mark.parametrize( + "request_creds", + [ + { + "langfuse_public_key": "pk-attacker", + "langfuse_secret_key": "sk-attacker", + "langfuse_host": "https://attacker.example", + }, + {"arize_api_key": "K-attacker", "arize_space_id": "S-attacker"}, + {"wandb_api_key": "w-attacker", "weave_endpoint": "https://attacker/otel"}, + ], +) +def test_request_credentials_are_inert_on_v2(request_creds): + """Any backend's credentials in the request's dynamic params (no admin + destinations) produce no per-tenant routing.""" + event = LLMCallEvent.from_dict( + { + "standard_callback_dynamic_params": request_creds, + "call_type": "acompletion", + "model": "gpt-4o", + } + ) + assert event.otel_destinations == () + cache = _cache("langfuse_otel") + default = NoOpTracer() + assert cache.tracer_for(default, event.otel_destinations) is default + assert cache._providers == {} + + +def test_admin_destinations_route(): + event = _event( + [ + { + "callback_name": "langfuse_otel", + "endpoint": "https://cloud.langfuse.com/api/public/otel", + "headers": {"Authorization": "Basic ADMIN"}, + } + ] + ) + assert len(event.otel_destinations) == 1 + cache = _cache("langfuse_otel") + cache.tracer_for(NoOpTracer(), event.otel_destinations) + assert len(cache._providers) == 1 + + +# --- a logger only exports the destinations tagged with its own backend ----- # + + +def test_logger_filters_destinations_to_its_backend(): + from litellm.integrations.otel.logger import OpenTelemetryV2 + + event = _event( + [ + { + "callback_name": "langfuse_otel", + "endpoint": "https://lf/api/public/otel", + "headers": {"Authorization": "Basic A"}, + }, + { + "callback_name": "arize", + "endpoint": "https://otlp.arize.com/v1", + "headers": {"space_id": "S"}, + }, + ] + ) + + class _Shim: + callback_name = "langfuse_otel" + + got = OpenTelemetryV2._destinations_for_backend(_Shim(), event) + assert [d.endpoint for d in got] == ["https://lf/api/public/otel"] + + +# --- multi-destination, same-backend: group by Resource so each gets its span -- # +# +# A backend that selects its target FROM the Resource (Arize's project) needs a +# differently-tagged span per destination, because a span carries exactly one +# Resource (a TracerProvider property). Pre-fix the gen-AI clone folded every +# destination into ONE last-wins Resource, so two Arize projects collapsed to one +# and only that project received the gen-AI span. ``tracers_for`` groups by +# ``destination_resource_attrs`` and returns one tracer (one provider, one Resource) +# per distinct group. Header-routed backends declare no Resource attrs, so they stay +# in one group with multiple exporters and keep routing by per-exporter auth. + + +def _arize_dest(project, space="S", key="K"): + return OtelDestination( + endpoint="https://otlp.arize.com/v1", + headers={"space_id": space, "api_key": key}, + callback_name="arize", + resource_attributes={"model_id": project, "arize.project.name": project}, + ) + + +def _provider_project(provider): + return provider.resource.attributes.get("arize.project.name") + + +def test_tracers_for_empty_returns_default_only(): + cache = _cache("arize") + default = NoOpTracer() + assert cache.tracers_for(default, ()) == (default,) + assert cache._providers == {} + + +def test_tracers_for_single_destination_one_group(): + """No regression: a single Arize destination yields one tracer/provider carrying + its project (same as the old single-merged path).""" + cache = _cache("arize") + tracers = cache.tracers_for(NoOpTracer(), (_arize_dest("solo"),)) + assert len(tracers) == 1 + assert {_provider_project(p) for p in cache._providers.values()} == {"solo"} + + +def test_tracers_for_two_arize_projects_split_into_separate_groups(): + """The fix: two Arize destinations with different Resource attrs must NOT + last-wins merge -- each project gets its own provider/Resource so each receives a + correctly-tagged gen-AI span.""" + cache = _cache("arize") + tracers = cache.tracers_for( + NoOpTracer(), (_arize_dest("projA"), _arize_dest("projB")) + ) + assert len(tracers) == 2 # one tracer per project group + assert {_provider_project(p) for p in cache._providers.values()} == { + "projA", + "projB", + } + + +def test_two_arize_projects_each_provider_has_its_own_single_project(): + """Each group's Resource carries exactly its own project (not the other's, not a + merge).""" + cache = _cache("arize") + cache.tracers_for(NoOpTracer(), (_arize_dest("projA"), _arize_dest("projB"))) + by_project = { + _provider_project(p): p.resource.attributes for p in cache._providers.values() + } + assert by_project["projA"]["model_id"] == "projA" + assert by_project["projB"]["model_id"] == "projB" + + +def test_header_routed_destinations_stay_one_group_with_two_exporters(): + """Langfuse/Weave declare no Resource attrs, so two distinct destinations collapse + into ONE group (one provider) with one exporter each -- they route by per-exporter + auth, so no per-Resource split is needed or wanted.""" + cache = _cache( + "langfuse_otel", + exporters=[ + ExporterSpec(kind="otlp_http", endpoint="https://env/v1", owner=None) + ], + ) + tracers = cache.tracers_for( + NoOpTracer(), + (_dest("https://a/v1", "Basic A"), _dest("https://b/v1", "Basic B")), + ) + assert len(tracers) == 1 # single empty-Resource group + assert len(cache._providers) == 1 + (provider,) = cache._providers.values() + endpoints = " ".join( + str(getattr(getattr(sp, "span_exporter", None), "_endpoint", "")) + for sp in provider._active_span_processor._span_processors + ) + # global + both destinations all live on the one provider (OTLP normalizes the + # endpoint by appending /v1/traces, so match on the host+path prefix) + assert "https://a/v1" in endpoints and "https://b/v1" in endpoints + + +def test_base_exporters_attach_to_first_group_only(): + """When a backend splits into multiple Resource groups, the configured/global + exporters must ride exactly ONE group, so the global receives the gen-AI span once + rather than once per project.""" + cache = _cache( + "arize", + exporters=[ExporterSpec(kind="in_memory", endpoint=None, owner=None)], + ) + cache.tracers_for(NoOpTracer(), (_arize_dest("projA"), _arize_dest("projB"))) + base_counts = {} + for provider in cache._providers.values(): + project = _provider_project(provider) + base_counts[project] = sum( + type(getattr(sp, "span_exporter", sp)).__name__ == "InMemorySpanExporter" + for sp in provider._active_span_processor._span_processors + ) + # exactly one group carries the global in_memory exporter; the other carries none + assert sorted(base_counts.values()) == [0, 1] + + +def test_generic_backend_resolves_generic_destination(): + """A Generic OTLP destination (callback_name='generic') must be picked up by the + generic OpenTelemetryV2 logger, so its gen-AI span routes to the destination's + otel_endpoint. Regression: 'generic' had no preset, so no generic logger existed and + the gen-AI span was dropped (only proxy-internal spans fanned out).""" + from litellm.integrations.otel.logger import OpenTelemetryV2 + + event = _event( + [ + { + "callback_name": "generic", + "endpoint": "http://collector:4318", + "headers": {"x-tenant": "t1"}, + }, + { + "callback_name": "arize", + "endpoint": "https://otlp.arize.com/v1", + "headers": {"space_id": "S"}, + }, + ] + ) + + class _Shim: + callback_name = "generic" + + got = OpenTelemetryV2._destinations_for_backend(_Shim(), event) + assert [d.endpoint for d in got] == ["http://collector:4318"] diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py b/tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py new file mode 100644 index 00000000000..e1af18c2dbc --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py @@ -0,0 +1,348 @@ +"""Tests for the per-request tenant fan-out SpanProcessor (Hoist). + +The fan-out processor lives on the main v2 ``TracerProvider`` and forwards every +finished span to the admin-resolved per-tenant destinations carried on the +request's ``ContextVar``. The contract under test: + +- Spans on the main provider land at every per-tenant destination matching this + backend (the ``owner_callback_name``). +- When destinations is empty (an unassigned identity, the SDK path, a request + before the resolver ran), the processor is a no-op. +- Concurrent requests with different destinations stay isolated -- contextvars + scope per task, so one request's tenant doesn't receive another's spans. +- The processor caches one BatchSpanProcessor per ``(endpoint, headers)`` pair + and skips destinations whose ``callback_name`` doesn't match its owner. +""" + +import asyncio +from typing import Any + +import pytest + +pytest.importorskip("opentelemetry") + +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 + InMemorySpanExporter, +) +from opentelemetry.sdk.trace.export import SpanExporter # noqa: E402 + +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 +from litellm.integrations.otel.plumbing.context import ( # noqa: E402 + set_request_destinations, + _request_destinations, +) +from litellm.integrations.otel.plumbing.fan_out import ( # noqa: E402 + TenantFanOutSpanProcessor, + _processor_key, +) +from litellm.integrations.otel.plumbing.providers import ( # noqa: E402 + destination_resource_attrs, +) +from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402 + + +@pytest.fixture(autouse=True) +def _reset_request_destinations(): + _request_destinations.set(()) + yield + _request_destinations.set(()) + + +def _build_provider_with_fan_out( + owner: str, exporter: SpanExporter +) -> tuple[TracerProvider, TenantFanOutSpanProcessor]: + """Build a real TracerProvider with the in-memory exporter as the configured + backend, plus the fan-out processor wired in. The fan-out processor's + per-destination processors are real BatchSpanProcessors, replaced in the + test by an injected SimpleSpanProcessor against an in-memory exporter via + ``monkeypatch`` so the test can read what was forwarded.""" + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider = providers.build_tracer_provider( + cfg, exporter=exporter, tenant_fan_out_owner=owner + ) + fan_out = next( + p + for p in provider._active_span_processor._span_processors + if isinstance(p, TenantFanOutSpanProcessor) + ) + return provider, fan_out + + +def test_fan_out_forwards_span_to_matching_destination(monkeypatch): + """A span on the main provider lands at the per-tenant destination whose + callback_name matches the fan-out owner. Removing the fan-out branch makes + this test fail (the tenant exporter never sees the span).""" + main_exporter = InMemorySpanExporter() + provider, fan_out = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + tenant_exporter = InMemorySpanExporter() + + # Swap the lazy per-destination processor build for an in-memory one so we + # don't actually hit the network. + def _stub_processor_for(self, destination): + return SimpleSpanProcessor(tenant_exporter) + + monkeypatch.setattr( + TenantFanOutSpanProcessor, "_processor_for", _stub_processor_for + ) + + set_request_destinations( + ( + OtelDestination( + callback_name="langfuse_otel", + endpoint="https://cloud.langfuse.com/api/public/otel", + headers={"Authorization": "Bearer pk:sk"}, + ), + ) + ) + tracer = provider.get_tracer("test") + with tracer.start_as_current_span("auth /chat/completions"): + pass + + main_names = [s.name for s in main_exporter.get_finished_spans()] + tenant_names = [s.name for s in tenant_exporter.get_finished_spans()] + assert "auth /chat/completions" in main_names + assert "auth /chat/completions" in tenant_names + + +def test_fan_out_forwards_proxy_internal_spans_to_every_destination(monkeypatch): + """Proxy-internal spans (server, auth phase, postgres, post-call ledger) + are generic OTel semconv with no backend-specific vocabulary, so they ship + to every admin-resolved destination regardless of its ``callback_name``. + The owner discriminator is only used to skip the gen-AI span (the v2 logger + routes that itself through TenantTracerCache to avoid wrong-vocabulary + duplicates).""" + main_exporter = InMemorySpanExporter() + _provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + tenant_exporter = InMemorySpanExporter() + monkeypatch.setattr( + TenantFanOutSpanProcessor, + "_processor_for", + lambda self, destination: SimpleSpanProcessor(tenant_exporter), + ) + + # Destination is tagged for a DIFFERENT backend (``arize``) than the + # owner (``langfuse_otel``). Proxy-internal spans must still forward. + set_request_destinations( + ( + OtelDestination( + callback_name="arize", + endpoint="https://otlp.arize.com/v1", + headers={"api_key": "k", "space_id": "s"}, + ), + ) + ) + tracer = _provider.get_tracer("test") + with tracer.start_as_current_span("auth"): + pass + + names = [s.name for s in tenant_exporter.get_finished_spans()] + assert "auth" in names + + +def test_destination_resource_attrs_reads_declared_attrs(monkeypatch): + """``destination_resource_attrs`` is backend-agnostic: it returns exactly what + the destination's builder declared, never consulting env itself. The env-vs- + credential precedence is enforced at build time (see test_presets_destinations), + so setting ARIZE_PROJECT_NAME here must NOT override the destination's own attrs.""" + monkeypatch.setenv("ARIZE_PROJECT_NAME", "global-project") + destination = OtelDestination( + callback_name="arize", + endpoint="https://otlp.arize.com/v1", + headers={"api_key": "k", "space_id": "s"}, + resource_attributes={ + "model_id": "tenant-project", + "arize.project.name": "tenant-project", + }, + ) + + assert destination_resource_attrs(destination) == { + "model_id": "tenant-project", + "arize.project.name": "tenant-project", + } + + +def test_destination_resource_attrs_empty_for_header_routed_backend(): + """A langfuse/weave-style destination routes by header and declares no Resource + attrs; the helper returns {} and the fan-out leaves the span Resource untouched.""" + destination = OtelDestination( + callback_name="langfuse_otel", + endpoint="https://cloud.langfuse.com/api/public/otel", + headers={"Authorization": "Basic x"}, + ) + assert destination_resource_attrs(destination) == {} + + +def test_fan_out_skips_genai_span_to_avoid_double_export(monkeypatch): + """The gen-AI LLM-call span is routed by the v2 logger through + ``TenantTracerCache`` to per-tenant exporters with the right attribute + mapper. Forwarding it here too would deliver a duplicate with the wrong + vocabulary, surfacing in the destination as an orphaned second span.""" + main_exporter = InMemorySpanExporter() + _provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + tenant_exporter = InMemorySpanExporter() + monkeypatch.setattr( + TenantFanOutSpanProcessor, + "_processor_for", + lambda self, destination: SimpleSpanProcessor(tenant_exporter), + ) + + set_request_destinations( + ( + OtelDestination( + callback_name="arize", + endpoint="https://otlp.arize.com/v1", + headers={}, + ), + ) + ) + tracer = _provider.get_tracer("test") + # A gen-AI span sets ``gen_ai.operation.name`` (the v2 emitter does this + # via the SpanRole.LLM_CALL mapper). Fake it explicitly here. + with tracer.start_as_current_span("chat gpt-4o") as span: + span.set_attribute("gen_ai.operation.name", "chat") + # A proxy-internal span also fires. + with tracer.start_as_current_span("auth /v1/chat/completions"): + pass + + names = [s.name for s in tenant_exporter.get_finished_spans()] + assert "auth /v1/chat/completions" in names + assert "chat gpt-4o" not in names # gen-AI skipped + + +def test_fan_out_noop_when_no_destinations(monkeypatch): + """An unassigned identity / pre-auth / SDK path leaves the contextvar at + its empty-tuple default; the processor must short-circuit, NOT crash, NOT + forward.""" + main_exporter = InMemorySpanExporter() + provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + forwarded: list[Any] = [] + monkeypatch.setattr( + TenantFanOutSpanProcessor, + "_processor_for", + lambda self, destination: forwarded.append(destination) or None, + ) + tracer = provider.get_tracer("test") + with tracer.start_as_current_span("auth"): + pass + assert forwarded == [] + + +def test_fan_out_caches_processor_per_destination_key(monkeypatch): + """Two requests with the same destination must share one cached + BatchSpanProcessor; two different destinations must build two. Otherwise + every request rebuilds the OTLP exporter (and its background thread).""" + main_exporter = InMemorySpanExporter() + _provider, fan_out = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + built: list[OtelDestination] = [] + + real_processor_for = TenantFanOutSpanProcessor._processor_for + + def _spy(self, destination): + result = real_processor_for(self, destination) + if result is not None: + built.append(destination) + return result + + # Build always returns None to keep the test offline, but we tracked + # invocations via the spy above. Easier: stub the inner builder. + def _stub(self, destination): + built.append(destination) + key = _processor_key(destination) + if key in self._processors: + return self._processors[key] + proc = SimpleSpanProcessor(InMemorySpanExporter()) + self._processors[key] = proc + return proc + + monkeypatch.setattr(TenantFanOutSpanProcessor, "_processor_for", _stub) + + dest_a = OtelDestination( + callback_name="langfuse_otel", endpoint="https://a/", headers={"k": "1"} + ) + dest_b = OtelDestination( + callback_name="langfuse_otel", endpoint="https://b/", headers={"k": "1"} + ) + set_request_destinations((dest_a,)) + tracer = _provider.get_tracer("test") + with tracer.start_as_current_span("s1"): + pass + set_request_destinations((dest_a,)) + with tracer.start_as_current_span("s2"): + pass + set_request_destinations((dest_b,)) + with tracer.start_as_current_span("s3"): + pass + + # _stub appends every call; the cache size is what matters: two unique + # ``(endpoint, headers)`` pairs -> two cached processors. + assert len(fan_out._processors) == 2 + + +def test_fan_out_per_request_isolation_with_concurrent_tasks(monkeypatch): + """Two requests running concurrently with different destinations must each + see only THEIR destination's spans. The contextvar scopes per asyncio task, + so this is a pin on the contextvar approach: switching to a global variable + would break this test.""" + main_exporter = InMemorySpanExporter() + provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + exporter_a = InMemorySpanExporter() + exporter_b = InMemorySpanExporter() + + def _stub(self, destination): + if destination.endpoint == "https://a/": + return SimpleSpanProcessor(exporter_a) + return SimpleSpanProcessor(exporter_b) + + monkeypatch.setattr(TenantFanOutSpanProcessor, "_processor_for", _stub) + tracer = provider.get_tracer("test") + + async def fire(label: str, endpoint: str): + set_request_destinations( + ( + OtelDestination( + callback_name="langfuse_otel", + endpoint=endpoint, + headers={}, + ), + ) + ) + with tracer.start_as_current_span(label): + await asyncio.sleep(0) + + async def driver(): + await asyncio.gather( + fire("req_a", "https://a/"), + fire("req_b", "https://b/"), + ) + + asyncio.run(driver()) + + names_a = {s.name for s in exporter_a.get_finished_spans()} + names_b = {s.name for s in exporter_b.get_finished_spans()} + assert "req_a" in names_a and "req_b" not in names_a + assert "req_b" in names_b and "req_a" not in names_b + + +def test_fan_out_owner_set_by_build_tracer_provider(): + """The v2 logger opts the MAIN provider into fan-out by passing its + callback_name; the TenantTracerCache clone providers do NOT, so the gen-AI + span emitted through them is not also forwarded by the fan-out processor. + Regressing this (adding the processor to clones) would double-export every + gen-AI span.""" + from litellm.integrations.otel.logger import OpenTelemetryV2 + + logger = OpenTelemetryV2(callback_name="arize") + main_procs = logger._tracer_provider._active_span_processor._span_processors + assert any(isinstance(p, TenantFanOutSpanProcessor) for p in main_procs) + # Build a tenant clone provider and confirm it has no fan-out processor. + clone = providers.build_tracer_provider(logger.config) + clone_procs = clone._active_span_processor._span_processors + assert not any(isinstance(p, TenantFanOutSpanProcessor) for p in clone_procs) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 0ceb7efbe0b..96276f30d7c 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -478,7 +478,7 @@ def test_live_llm_span_anchors_to_root_with_no_active_span(): set_request_root_span(server) kwargs = _kwargs() logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) - assert logger._open_llm_calls["call_1"].span is not None # live, via anchor + assert logger._open_llm_calls["call_1"].spans # live, via anchor asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) server.end() by_name = {s.name: s for s in exporter.get_finished_spans()} @@ -498,7 +498,7 @@ def test_deferred_llm_span_reads_anchor_at_close(): kwargs = _kwargs() # pre_call with NO anchor and no active span → deferred. logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) - assert logger._open_llm_calls["call_1"].span is None # deferred + assert logger._open_llm_calls["call_1"].spans == () # deferred # Anchor becomes visible at close (worker copied the request task's context). set_request_root_span(server) asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) @@ -538,6 +538,163 @@ def test_synthetic_error_log_produces_no_llm_span(): assert "auth /chat/completions" in names # auth span itself still recorded +def test_lazy_activation_emits_llm_span_when_destination_resolves(monkeypatch): + """LIT-3850 lazy-activation seam: a v2 instance born inside the success path + (because the destination resolver appended its backend to ``success_callback`` + on this request) was not in the callback list when ``pre_call`` iterated, so + no carrier was opened. The close must still emit the gen-ai span when the + payload is present and the admin-resolved destinations name this backend, so + the per-tenant exporter ships it. Without the fallthrough, this test sees + zero spans.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + # Make ``tracers_for`` return the logger's default tracer regardless of the + # destinations passed: the test asserts the close-path emitted the span, not + # that the per-destination provider clone wired up an OTLP exporter (the + # routing cache's job, covered separately). The default tracer is bound to + # the in-memory exporter so the test can read the result. + tracer_for_calls: list[tuple] = [] + + def _fake_tracers_for(default, destinations): + tracer_for_calls.append(destinations) + return (default,) + + monkeypatch.setattr(logger._tenant_tracers, "tracers_for", _fake_tracers_for) + kwargs = _kwargs() + kwargs["standard_callback_dynamic_params"] = { + "otel_destinations": [ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {"api_key": "k"}, + } + ] + } + assert "call_1" not in logger._open_llm_calls # no carrier opened + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + names = [s.name for s in exporter.get_finished_spans()] + assert "chat gpt-4o" in names + # ``tracers_for`` was invoked with exactly the resolved destination, proving + # the deferred path used per-tenant routing rather than the default tracer + # blindly. + assert len(tracer_for_calls) == 1 + (dests,) = tracer_for_calls + assert len(dests) == 1 and dests[0].endpoint == "https://otlp.example.com/v1" + + +def test_second_close_after_opened_call_does_not_emit_duplicate(monkeypatch): + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, destinations: (default,), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + kwargs["standard_callback_dynamic_params"] = { + "otel_destinations": [ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {"api_key": "k"}, + } + ] + } + + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + server.end() + + names = [s.name for s in exporter.get_finished_spans()] + assert names.count("chat gpt-4o") == 1 + + +def test_close_without_carrier_and_without_destination_drops_silently(): + """The pre-existing early-return semantics (auth gate / pre-call guardrail + rejection with no destination resolving to this backend) must be preserved: + no phantom span. The fix only widens emit-on-close when the admin-resolved + destinations name this backend AND the payload exists.""" + logger, exporter = _logger() + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + asyncio.run(logger.async_log_success_event(_kwargs(), None, None, None)) + server.end() + names = [s.name for s in exporter.get_finished_spans()] + assert "chat gpt-4o" not in names + + +def test_close_without_carrier_drops_when_payload_missing(monkeypatch): + """No carrier + no payload = the auth-gate rejection case (no upstream call + happened). Must drop even when destinations resolve, so a phantom span is + never emitted for a request the gate refused.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + kwargs = { + "litellm_params": {"metadata": {}}, + "standard_callback_dynamic_params": { + "otel_destinations": [ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {}, + } + ] + }, + } + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + assert exporter.get_finished_spans() == () + + +def test_close_dedupes_duplicate_callbacks(monkeypatch): + """Cursor BugBot regression: a normal close pops the carrier and emits + the LLM span; a second callback for the same call_id must not emit a + duplicate. Before the dedup guard, the second close hit the + carrier-is-None branch and fired _emit_deferred_llm_call again whenever + payload + destinations remained on the kwargs, double-exporting the + span (e.g. success + failure callbacks both firing, or a custom callback + fanning out).""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + monkeypatch.setattr( + logger._tenant_tracers, "tracers_for", lambda default, dests: (default,) + ) + kwargs = _kwargs() + kwargs["standard_callback_dynamic_params"] = { + "otel_destinations": [ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {}, + } + ] + } + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + # Second callback for the same call_id: payload + destinations still on + # kwargs, but no second span must emit. + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + server.end() + llm_spans = [s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"] + assert len(llm_spans) == 1 + + def test_create_request_started_span_captures_anchor(): """``create_litellm_proxy_request_started_span`` doubles as the anchor capture point: the active server span becomes the request root for later spans.""" @@ -1580,3 +1737,139 @@ def test_metrics_disabled_by_default_records_nothing(monkeypatch): ) ) assert _emitted_metric_names(reader) == set() + + +def _second_group_tracer(logger): + """A second independent in-memory provider standing in for a second Resource group + (e.g. a second Arize project); returns (tracer, exporter).""" + exporter = InMemorySpanExporter() + provider = providers.build_tracer_provider(logger.config, exporter=exporter) + return provider.get_tracer("litellm"), exporter + + +def test_genai_span_emitted_to_every_group_live(monkeypatch): + """Multi-destination fix (live path): when ``tracers_for`` returns two tracers (two + Resource groups, e.g. two Arize projects), the gen-AI span opened at ``pre_call`` + must be opened+finished on BOTH -- the bug was only one project receiving it.""" + logger, exporter_a = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + tracer_b, exporter_b = _second_group_tracer(logger) + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, dests: (default, tracer_b), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + kwargs["standard_callback_dynamic_params"] = { + "otel_destinations": [ + {"callback_name": "in_memory", "endpoint": "https://x/v1", "headers": {}} + ] + } + _emit_llm(logger, kwargs, ambient=server) + server.end() + assert [s.name for s in exporter_a.get_finished_spans()].count("chat gpt-4o") == 1 + assert [s.name for s in exporter_b.get_finished_spans()].count("chat gpt-4o") == 1 + + +def test_genai_span_emitted_to_every_group_deferred(monkeypatch): + """Same fix, deferred path (no carrier at ``pre_call``): ``emit_fanout`` dedups once + on the call id then emits the span on every group's tracer, so both projects get + exactly one.""" + logger, exporter_a = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + tracer_b, exporter_b = _second_group_tracer(logger) + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, dests: (default, tracer_b), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + kwargs["standard_callback_dynamic_params"] = { + "otel_destinations": [ + {"callback_name": "in_memory", "endpoint": "https://x/v1", "headers": {}} + ] + } + # no pre_call -> no carrier -> deferred close path + assert "call_1" not in logger._open_llm_calls + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + assert [s.name for s in exporter_a.get_finished_spans()].count("chat gpt-4o") == 1 + assert [s.name for s in exporter_b.get_finished_spans()].count("chat gpt-4o") == 1 + + +def _generic_dest(): + return { + "callback_name": "generic", + "endpoint": "http://collector:4318", + "headers": {}, + } + + +def test_generic_destination_emits_genai_span(monkeypatch): + """Acceptance #1: a request whose only admin destination is a Generic OTLP + destination emits the chat gen-AI span (regression: 'generic' had no preset, + so the gen-AI span was dropped and only proxy-internal spans reached the endpoint). + The destination's callback_name='generic' is matched by the generic logger, and the + span is routed through the per-destination tracer.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "generic") + routed: list = [] + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, dests: (routed.append(dests) or (default,)), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + kwargs["standard_callback_dynamic_params"] = { + "otel_destinations": [_generic_dest()] + } + _emit_llm(logger, kwargs, ambient=server) + server.end() + assert "chat gpt-4o" in [s.name for s in exporter.get_finished_spans()] + # the gen-AI span was routed for the generic destination (not an empty/global set) + assert any( + len(d) == 1 and d[0].endpoint == "http://collector:4318" for d in routed + ), "generic destination was not routed to the generic logger's tracer" + + +def test_generic_destination_emits_error_span_on_failure(monkeypatch): + """Acceptance #3: a FAILED call to a Generic OTLP destination still emits the + chat span, with OTEL status ERROR and the exception type.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "generic") + monkeypatch.setattr( + logger._tenant_tracers, "tracers_for", lambda default, dests: (default,) + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + payload = _payload( + status="failure", + error_information={ + "error_class": "AuthenticationError", + "error_message": "Incorrect API key", + }, + ) + kwargs = _kwargs(payload=payload) + kwargs["standard_callback_dynamic_params"] = { + "otel_destinations": [_generic_dest()] + } + _emit_llm(logger, kwargs, ambient=server, fail=True) + server.end() + spans = {s.name: s for s in exporter.get_finished_spans()} + assert "chat gpt-4o" in spans + assert spans["chat gpt-4o"].status.status_code is StatusCode.ERROR + assert spans["chat gpt-4o"].attributes["error.type"] == "AuthenticationError" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py index e879766c5c7..f843a8de8c7 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py @@ -87,3 +87,33 @@ def test_config_folds_legacy_exporter_triple_into_exporters_list(): assert len(cfg.exporters) == 1 assert cfg.exporters[0].kind == "otlp_http" assert cfg.exporters[0].endpoint == "https://api.example.com" + + +def _processor_names(provider): + return [type(p).__name__ for p in provider._active_span_processor._span_processors] + + +def test_main_logger_provider_always_carries_fan_out(): + # The FastAPI server span (and auth/db/cost spans) bind to the main v2 logger's + # provider. That provider MUST carry the TenantFanOutSpanProcessor even for the + # generic global logger (no backend named) -- otherwise, for an admin-owned- + # destination-only deployment, the server span never reaches the destination and + # the gen-AI child is orphaned. Regression for the orphaned-span RCA. + from litellm.integrations.otel.logger import OpenTelemetryV2 + + generic = OpenTelemetryV2() + assert "TenantFanOutSpanProcessor" in _processor_names(generic._tracer_provider) + + named = OpenTelemetryV2(callback_name="arize") + assert "TenantFanOutSpanProcessor" in _processor_names(named._tracer_provider) + + +def test_build_tracer_provider_attach_fan_out_flag(): + cfg = OpenTelemetryV2Config(exporter="console") + # Per-tenant clone providers pass neither flag and must NOT fan out (else the + # gen-AI span they carry would be double-exported). + clone = build_tracer_provider(cfg) + assert "TenantFanOutSpanProcessor" not in _processor_names(clone) + # The main provider opts in explicitly. + main = build_tracer_provider(cfg, attach_tenant_fan_out=True) + assert "TenantFanOutSpanProcessor" in _processor_names(main) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_presets.py b/tests/test_litellm/integrations/otel/test_otel_v2_presets.py index add2caf9a48..b14128bf384 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_presets.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_presets.py @@ -33,27 +33,72 @@ def test_agentops_preset_does_no_network_io(monkeypatch): assert spec.options == {"api_key": "ak-123"} # carried to the lazy exporter -def test_agentops_preset_without_key_omits_options(monkeypatch): +def test_agentops_preset_without_key_omits_exporter(monkeypatch): + # With no API key the lazy-auth exporter has nothing to mint a JWT from, so + # the preset must not contribute a global exporter at all (it would otherwise + # fail every export). Admin-owned destinations carry their own credentials. monkeypatch.delenv("AGENTOPS_API_KEY", raising=False) cfg = agentops_preset() - spec = next(e for e in cfg.exporters if e.kind == _AGENTOPS_EXPORTER_KIND) - assert spec.options is None + assert [e for e in cfg.exporters if e.kind == _AGENTOPS_EXPORTER_KIND] == [] + + +def test_arize_preset_without_credentials_omits_exporter(monkeypatch): + # Arize's OTLP ingestion rejects unauthenticated exports (PERMISSION_DENIED), + # so with no Arize credentials the preset must not contribute a credential-less + # global exporter pointed at the Arize cloud. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.arize import arize_preset + + for var in ( + "ARIZE_SPACE_ID", + "ARIZE_SPACE_KEY", + "ARIZE_API_KEY", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + ): + monkeypatch.delenv(var, raising=False) + cfg = arize_preset() + assert [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_AX] == [] + + monkeypatch.setenv("ARIZE_SPACE_ID", "S") + monkeypatch.setenv("ARIZE_API_KEY", "K") + cfg = arize_preset() + assert [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_AX] != [] + + +def test_phoenix_preset_without_config_omits_exporter(monkeypatch): + # Unconfigured Phoenix defaults to http://localhost:6006; the preset must not + # contribute that exporter unless Phoenix is actually configured (cloud key or + # collector endpoint), so admin-owned-only setups don't export to localhost. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.phoenix import ( + _PHOENIX_ENV_VARS, + phoenix_preset, + ) + + for var in _PHOENIX_ENV_VARS: + monkeypatch.delenv(var, raising=False) + cfg = phoenix_preset() + assert [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_PHOENIX] == [] + + monkeypatch.setenv("PHOENIX_API_KEY", "px-key") + cfg = phoenix_preset() + assert [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_PHOENIX] != [] def test_agentops_exporter_factory_is_registered(): assert _AGENTOPS_EXPORTER_KIND in providers._EXPORTER_FACTORIES -def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch): - """Each dynamic-credential preset must tag the exporter it contributes with - its own callback name, so per-request tenant routing - (``TenantTracerCache``) applies that integration's credentials only to its - own exporter and never bleeds them onto a co-configured backend. +def test_destination_routable_presets_tag_exporter_with_matching_owner(monkeypatch): + """Each destination-routable preset must tag the exporter it contributes with + its own callback name, so per-tenant routing (``TenantTracerCache``) points + that integration's admin destination at its own exporter only and never + rewrites a co-configured backend's exporter. """ - from litellm.integrations.otel.presets import ( - DYNAMIC_HEADERS_BY_CALLBACK, - PRESET_BY_CALLBACK, + from litellm.integrations.otel.presets.destinations import ( + OTEL_V2_DESTINATION_CALLBACKS, ) + from litellm.integrations.otel.presets import PRESET_BY_CALLBACK monkeypatch.setenv("ARIZE_SPACE_ID", "S") monkeypatch.setenv("ARIZE_API_KEY", "K") @@ -65,7 +110,7 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch): from litellm.integrations.otel.model.config import ExporterOwner - for callback_name in DYNAMIC_HEADERS_BY_CALLBACK: + for callback_name in OTEL_V2_DESTINATION_CALLBACKS: cfg = PRESET_BY_CALLBACK[callback_name]() owners = {e.owner for e in cfg.exporters} assert ExporterOwner(callback_name) in owners, ( @@ -160,3 +205,83 @@ def test_agentops_endpoint_points_at_live_host(): # so a typo or stale domain can never ship again. assert _AGENTOPS_ENDPOINT == "https://otlp.agentops.ai/v1/traces" assert "agentops.cloud" not in _AGENTOPS_ENDPOINT + + +def test_credential_mandatory_presets_raise_without_creds_by_default(monkeypatch): + # weave/langfuse/levo are credential-mandatory: with no global env creds and no + # opt-in to degrade, the preset must RAISE so a misconfigured global callback + # (e.g. ``callbacks: ["weave_otel"]`` with no keys) fails loud at startup, the + # same error story as before V2 landed. _maybe_construct_otel_v2 relies on this + # raise to defer to the legacy path. + from litellm.integrations.otel.presets.langfuse import langfuse_preset + from litellm.integrations.otel.presets.levo import levo_preset + from litellm.integrations.otel.presets.weave import weave_preset + + for var in ( + "WANDB_API_KEY", + "WANDB_PROJECT_ID", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LEVOAI_API_KEY", + ): + monkeypatch.delenv(var, raising=False) + + for preset in (weave_preset, langfuse_preset, levo_preset): + with pytest.raises(Exception): + preset() + + +def test_credential_mandatory_presets_degrade_when_allowed(monkeypatch): + # When an admin-owned destination is the reason for construction it carries its + # own per-tenant credentials, so the preset is called with + # allow_missing_credentials=True and must degrade to an exporter-less config + # (keeping its mappers) instead of raising -- otherwise the destination never gets + # a v2 logger and its gen-AI span falls to the generic global logger. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.langfuse import langfuse_preset + from litellm.integrations.otel.presets.levo import levo_preset + from litellm.integrations.otel.presets.weave import weave_preset + + for var in ( + "WANDB_API_KEY", + "WANDB_PROJECT_ID", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LEVOAI_API_KEY", + ): + monkeypatch.delenv(var, raising=False) + + for preset, owner in ( + (weave_preset, ExporterOwner.WEAVE_OTEL), + (langfuse_preset, ExporterOwner.LANGFUSE_OTEL), + (levo_preset, ExporterOwner.LEVO), + ): + cfg = preset(allow_missing_credentials=True) # must not raise + assert [e for e in cfg.exporters if e.owner == owner] == [] + + +def test_weave_preset_with_creds_contributes_exporter(monkeypatch): + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.weave import weave_preset + + monkeypatch.setenv("WANDB_API_KEY", "w-key") + monkeypatch.setenv("WANDB_PROJECT_ID", "entity/project") + cfg = weave_preset() + assert [e for e in cfg.exporters if e.owner == ExporterOwner.WEAVE_OTEL] != [] + + +def test_generic_preset_needs_no_global_env_and_emits_genai(monkeypatch): + # Acceptance #2: the generic preset is vendor-neutral and admin-destination-only -- + # it must build with NO global OTEL env vars, never raise, carry the standard genai + # (+legacy) mappers, and contribute NO vendor exporter (the per-destination exporter + # is appended by the router). + from litellm.integrations.otel.presets.generic import generic_preset + + for var in ("OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_ENDPOINT", "OTEL_EXPORTER"): + monkeypatch.delenv(var, raising=False) + cfg = generic_preset() # must not raise, no env needed + assert "genai" in cfg.mapper_names + # no vendor (owned) exporter contributed -- only the base/global passthrough, if any + assert all(e.owner is None for e in cfg.exporters) + # the degrade flag is accepted (Preset protocol) and irrelevant -- still builds + assert generic_preset(allow_missing_credentials=True) is not None diff --git a/tests/test_litellm/integrations/otel/test_presets_destinations.py b/tests/test_litellm/integrations/otel/test_presets_destinations.py new file mode 100644 index 00000000000..25b06b36893 --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_presets_destinations.py @@ -0,0 +1,212 @@ +"""``build_destination`` maps an admin credential to a generic OTLP destination. + +The point of these tests is that the resolution is backend-agnostic: Langfuse, +Arize, Weave, and any raw collector all resolve to an ``{endpoint, headers}`` +the router exports through, and an incomplete credential resolves to nothing. +""" + +import base64 +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.integrations.otel.presets.destinations import ( + OTEL_V2_DESTINATION_CALLBACKS, + build_destination, +) + + +def test_langfuse_endpoint_derived_from_host_with_basic_auth(): + dest = build_destination( + "langfuse_otel", + { + "langfuse_host": "https://cloud.langfuse.com", + "langfuse_public_key": "pk-eu", + "langfuse_secret_key": "sk-eu", + }, + ) + assert dest is not None + assert dest.endpoint == "https://cloud.langfuse.com/api/public/otel" + scheme, b64 = dest.headers["Authorization"].split(" ", 1) + assert scheme == "Basic" + assert base64.b64decode(b64).decode() == "pk-eu:sk-eu" + + +def test_langfuse_bare_host_gets_https_and_path(): + dest = build_destination( + "langfuse_otel", + { + "langfuse_host": "my-langfuse.internal", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + }, + ) + assert dest is not None + assert dest.endpoint == "https://my-langfuse.internal/api/public/otel" + + +def test_langfuse_without_host_defaults_to_us_cloud(): + dest = build_destination( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + ) + assert dest is not None + assert dest.endpoint == "https://us.cloud.langfuse.com/api/public/otel" + + +def test_langfuse_incomplete_returns_none(): + assert build_destination("langfuse_otel", {"langfuse_public_key": "pk"}) is None + + +def test_arize_space_and_api_key_headers(monkeypatch): + monkeypatch.delenv("ARIZE_PROJECT_NAME", raising=False) + dest = build_destination("arize", {"arize_space_id": "S", "arize_api_key": "K"}) + assert dest is not None + assert dest.endpoint == "https://otlp.arize.com/v1" + assert dest.headers == {"space_id": "S", "api_key": "K"} + + +# --- per-backend Resource declaration -------------------------------------- # +# +# Each backend declares the Resource attributes its ingestion needs, in its own +# builder. Arize is the only first-class backend that routes by a Resource +# attribute (``model_id``); langfuse / weave / generic route by auth header and +# declare none. The shared ``destination_resource_attrs`` just reads whatever the +# builder put on the destination, so the model generalizes: a new backend that +# needs Resource-level routing only populates ``resource_attributes`` here. + + +def test_arize_project_from_credential_sets_resource_attrs(monkeypatch): + monkeypatch.delenv("ARIZE_PROJECT_NAME", raising=False) + dest = build_destination( + "arize", + {"arize_space_id": "S", "arize_api_key": "K", "arize_project_name": "team-x"}, + ) + assert dest is not None + assert dest.resource_attributes == { + "model_id": "team-x", + "arize.project.name": "team-x", + } + + +def test_arize_project_falls_back_to_env(monkeypatch): + monkeypatch.setenv("ARIZE_PROJECT_NAME", "env-proj") + dest = build_destination("arize", {"arize_space_id": "S", "arize_api_key": "K"}) + assert dest is not None + assert dest.resource_attributes == { + "model_id": "env-proj", + "arize.project.name": "env-proj", + } + + +def test_arize_credential_project_wins_over_env(monkeypatch): + monkeypatch.setenv("ARIZE_PROJECT_NAME", "env-proj") + dest = build_destination( + "arize", + { + "arize_space_id": "S", + "arize_api_key": "K", + "arize_project_name": "cred-proj", + }, + ) + assert dest is not None + assert dest.resource_attributes["model_id"] == "cred-proj" + + +def test_arize_no_project_anywhere_has_empty_resource_attrs(monkeypatch): + monkeypatch.delenv("ARIZE_PROJECT_NAME", raising=False) + dest = build_destination("arize", {"arize_space_id": "S", "arize_api_key": "K"}) + assert dest is not None + assert dest.resource_attributes == {} + + +def test_header_routed_backends_declare_no_resource_attrs(): + """langfuse / weave / generic route the project via auth headers, so they + declare no Resource attributes -- the generalization counterpart to arize.""" + langfuse = build_destination( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + ) + weave = build_destination( + "weave_otel", + { + "wandb_api_key": "w", + "weave_endpoint": "https://trace.wandb.ai/otel/v1/traces", + }, + ) + generic = build_destination( + "self_hosted", {"otel_endpoint": "https://collector:4318/v1/traces"} + ) + for dest in (langfuse, weave, generic): + assert dest is not None + assert dest.resource_attributes == {} + + +def test_weave_requires_only_api_key_and_defaults_endpoint(): + # No API key -> nothing. + assert build_destination("weave_otel", {}) is None + # The API key alone is enough: Weave cloud's endpoint is fixed, so it defaults + # to the cloud OTLP path and the endpoint field is optional. + dest = build_destination( + "weave_otel", + {"wandb_api_key": "w", "weave_project_id": "entity/project"}, + ) + assert dest is not None + assert dest.endpoint == "https://trace.wandb.ai/otel/v1/traces" + assert dest.headers["project_id"] == "entity/project" + assert "Authorization" in dest.headers + + +def test_generic_passthrough_covers_any_backend(): + dest = build_destination( + "some_self_hosted_collector", + { + "otel_endpoint": "https://collector.internal:4318/v1/traces", + "otel_headers": "x-api-key=abc,x-team=42", + }, + ) + assert dest is not None + assert dest.endpoint == "https://collector.internal:4318/v1/traces" + assert dest.headers == {"x-api-key": "abc", "x-team": "42"} + + +def test_unknown_backend_without_generic_fields_returns_none(): + assert build_destination("mystery", {"foo": "bar"}) is None + + +def test_registry_lists_the_first_class_backends(): + assert OTEL_V2_DESTINATION_CALLBACKS == frozenset( + {"langfuse_otel", "arize", "weave_otel"} + ) + + +def test_endpoint_whitespace_is_trimmed(): + # A stray leading/trailing space in the endpoint (an easy create-form slip) + # makes a malformed OTLP URL the exporter rejects with a 404, so values are + # trimmed before the destination is built. + dest = build_destination( + "some_collector", + {"otel_endpoint": " https://collector.internal:4318/v1/traces "}, + ) + assert dest is not None + assert dest.endpoint == "https://collector.internal:4318/v1/traces" + + +def test_weave_endpoint_completed_to_otel_path(): + # Weave's OTLP path is /otel/v1/traces, not the bare /v1/traces the generic + # exporter would append; a host must be completed here or the export 404s. + # Idempotent when the full path or the /otel prefix is already supplied. + for given, expected in ( + ("https://trace.wandb.ai", "https://trace.wandb.ai/otel/v1/traces"), + ("https://trace.wandb.ai/otel", "https://trace.wandb.ai/otel/v1/traces"), + ( + "https://trace.wandb.ai/otel/v1/traces", + "https://trace.wandb.ai/otel/v1/traces", + ), + ): + dest = build_destination( + "weave_otel", {"wandb_api_key": "w", "weave_endpoint": given} + ) + assert dest is not None + assert dest.endpoint == expected diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 7ffd09b931f..2e7424526c8 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -1928,12 +1928,21 @@ class TestOpenTelemetryEndpointNormalization(unittest.TestCase): "https://example.com/prefix/v2/trace/otlp", "https://example.com/prefix/v2/trace/otlp", ), + ( + "https://app.langtrace.ai/api/trace", + "https://app.langtrace.ai/api/trace", + ), + ( + "https://app.langtrace.ai/api/trace/", + "https://app.langtrace.ai/api/trace", + ), ] ) def test_normalize_traces_nonstandard_otlp_ingest_urls_unchanged( self, input_url: str, expected: str ) -> None: - """Splunk-style /v2/trace/otlp endpoints must not get /v1/traces appended.""" + """Vendor full-path ingest URLs (Splunk /v2/trace/otlp, Langtrace /api/trace) + must not get /v1/traces appended — that 404s.""" otel = OpenTelemetry() self.assertEqual( otel._normalize_otel_endpoint(input_url, "traces"), @@ -5852,3 +5861,45 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): exporter="console", attributes=attributes ) ) + + +class LangtraceV1ConfigTest(unittest.TestCase): + def test_langtrace_uses_live_host_and_x_api_key_header(self): + """Regression: litellm hardcoded the stale https://langtrace.ai/api/trace (now 404) + with header ``api_key=``. The live ingest is https://app.langtrace.ai/api/trace and + the auth header is ``x-api-key``. Pins both so the stale values can't return.""" + import litellm + import litellm.litellm_core_utils.litellm_logging as ll + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + saved = { + k: os.environ.get(k) + for k in ( + "LANGTRACE_API_KEY", + "LITELLM_OTEL_V2", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + ) + } + try: + ll._in_memory_loggers.clear() # force a fresh build, not a cached logger + os.environ["LANGTRACE_API_KEY"] = "lt-test" + os.environ.pop("LITELLM_OTEL_V2", None) + is_otel_v2_enabled.cache_clear() + litellm.credential_list = [] + logger = ll._init_custom_logger_compatible_class("langtrace", None, None) + assert isinstance(logger, OpenTelemetry) + self.assertEqual( + logger.config.endpoint, "https://app.langtrace.ai/api/trace" + ) + self.assertEqual( + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"], "x-api-key=lt-test" + ) + finally: + is_otel_v2_enabled.cache_clear() + ll._in_memory_loggers.clear() + for key, value in saved.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index f63216b96d4..c5c57d82724 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -36,6 +36,58 @@ def test_resolves_plain_values_from_metadata(): assert params.get("langfuse_host") == "https://test.langfuse.com" +def test_otel_destinations_read_from_litellm_metadata_only(): + """The admin-resolved OTEL destinations are carried server-side under + ``litellm_metadata`` (a known internal key, scrubbed from the provider body) -- + NOT as a top-level kwarg, since an unknown top-level key would leak to providers. + They must surface on the dynamic params so the v2 logger can fan out to them.""" + destinations = [ + { + "callback_name": "langfuse_otel", + "endpoint": "https://cloud.langfuse.com/api/public/otel", + "headers": {"Authorization": "Basic ADMIN"}, + } + ] + + params = initialize_standard_callback_dynamic_params( + {"litellm_metadata": {"otel_destinations": destinations}} + ) + + assert params.get("otel_destinations") == destinations + + +def test_otel_destinations_top_level_kwarg_is_ignored(): + """A top-level ``otel_destinations`` kwarg is intentionally NOT read. The proxy + stashes admin-resolved destinations under ``litellm_metadata`` to keep unknown + keys out of the body forwarded to the provider; reading the top-level key would + re-open that surface and is therefore ignored.""" + params = initialize_standard_callback_dynamic_params( + {"otel_destinations": [{"callback_name": "langfuse_otel"}]} + ) + assert params.get("otel_destinations") is None + + +def test_otel_destinations_never_read_from_request_metadata(): + """A request body/metadata must not be able to inject OTEL destinations: + otel_destinations is deliberately absent from the request-read whitelist, so a + value nested in metadata is ignored. Guards the trust boundary.""" + kwargs = { + "metadata": { + "otel_destinations": [ + { + "callback_name": "langfuse_otel", + "endpoint": "https://attacker.example/api/public/otel", + "headers": {"Authorization": "Basic ATTACKER"}, + } + ] + } + } + + params = initialize_standard_callback_dynamic_params(kwargs) + + assert params.get("otel_destinations") is None + + def test_env_reference_at_top_level_raises_with_guidance(): kwargs = {"langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY"} diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 5a8ac0313a7..de98d354389 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3532,9 +3532,9 @@ async def test_streaming_anthropic_messages_openai_bridge_fires_success_logging( await _drain_until_logged(logger) assert chunks, "stream yielded no chunks" - assert any("content_block_delta" in _chunk_text(c) for c in chunks), ( - "no delta chunks surfaced; the streaming text deltas were not forwarded" - ) + assert any( + "content_block_delta" in _chunk_text(c) for c in chunks + ), "no delta chunks surfaced; the streaming text deltas were not forwarded" assert logger.success_payload is not None, ( "async_log_success_event never fired for streaming /v1/messages -> openai " "Responses bridge; the no-op stream path dropped the spend row" @@ -3585,3 +3585,123 @@ def test_failure_handler_zeroes_spend_without_recovered_usage(logging_obj): assert payload["status"] == "failure" assert payload["response_cost"] == 0 assert payload["total_tokens"] == 0 + + +def test_admin_owned_destination_uses_otel_v2_without_global_flag(monkeypatch): + # An admin-owned logging destination (a "logging" credential, created from the + # UI) must make the backend resolve to the OTEL v2 logger even when + # LITELLM_OTEL_V2 is off, so the per-destination credentials drive the export. + # Otherwise activation falls back to the legacy global logger, which ignores + # the destination's credentials and exports with absent global env creds. + from types import SimpleNamespace + + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + assert is_otel_v2_enabled() is False + + # No logging credential for the backend -> legacy fallback (None). + monkeypatch.setattr(litellm, "credential_list", []) + assert _maybe_construct_otel_v2("arize", []) is None + + # A logging destination registered for the backend -> v2 owns it. + monkeypatch.setattr( + litellm, + "credential_list", + [ + SimpleNamespace( + credential_name="arize-poc", + credential_info={ + "credential_type": "logging", + "description": "arize", + }, + ) + ], + ) + logger = _maybe_construct_otel_v2("arize", []) + is_otel_v2_enabled.cache_clear() + assert isinstance(logger, OpenTelemetryV2) + assert logger.callback_name == "arize" + + +def test_credential_mandatory_backend_global_misconfig_stays_loud(monkeypatch): + # Regression: weave/langfuse/levo are credential-mandatory; before V2 a global + # callback (e.g. ``callbacks: ["weave_otel"]``) with no credentials failed loud at + # startup. _maybe_construct_otel_v2 must preserve that: with V2 on but no admin-owned + # destination and no creds, the preset raises, _maybe_construct returns None, and the + # caller falls through to the legacy path that re-raises. An admin-owned destination, + # by contrast, carries its own per-tenant creds, so the same missing-global-creds + # state must degrade to a working v2 logger instead of silently producing nothing. + from types import SimpleNamespace + + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + for var in ("WANDB_API_KEY", "WANDB_PROJECT_ID"): + monkeypatch.delenv(var, raising=False) + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + # Global callback, no destination, no creds -> None so the caller fails loud. + monkeypatch.setattr(litellm, "credential_list", []) + assert _maybe_construct_otel_v2("weave_otel", []) is None + + # Admin-owned weave destination present -> degrade to a working v2 logger. + monkeypatch.setattr( + litellm, + "credential_list", + [ + SimpleNamespace( + credential_name="wb-poc", + credential_info={ + "credential_type": "logging", + "description": "weave_otel", + }, + ) + ], + ) + logger = _maybe_construct_otel_v2("weave_otel", []) + is_otel_v2_enabled.cache_clear() + assert isinstance(logger, OpenTelemetryV2) + assert logger.callback_name == "weave_otel" + + +def test_generic_admin_destination_builds_otel_v2_logger(monkeypatch): + # The Generic OTLP passthrough ('generic') must build an OpenTelemetryV2 logger when + # an admin-owned generic destination is registered (even with LITELLM_OTEL_V2 off), + # so the gen-AI span routes to the destination's otel_endpoint. Without a generic + # destination and without the global flag, it stays None (no generic logger). + from types import SimpleNamespace + + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + + monkeypatch.setattr(litellm, "credential_list", []) + assert _maybe_construct_otel_v2("generic", []) is None + + monkeypatch.setattr( + litellm, + "credential_list", + [ + SimpleNamespace( + credential_name="ui-generic", + credential_info={ + "credential_type": "logging", + "description": "generic", + }, + ) + ], + ) + logger = _maybe_construct_otel_v2("generic", []) + is_otel_v2_enabled.cache_clear() + assert isinstance(logger, OpenTelemetryV2) + assert logger.callback_name == "generic" diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 4141b1f20c3..d2225f36e72 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -3788,6 +3788,37 @@ async def test_builder_succeeds_when_db_lookup_returns_valid_token(): mock_return.assert_awaited_once() +@pytest.mark.asyncio +async def test_builder_hoists_destinations_before_post_lookup_auth_checks(): + valid_token = UserAPIKeyAuth(api_key="sk-db-lookup-test", token="hashed-valid") + get_key_object = AsyncMock(return_value=valid_token) + + async def _assert_hoisted_first(*args, **kwargs): + assert mock_hoist.await_count == 1 + + with ( + patch( + "litellm.proxy.auth.user_api_key_auth._return_user_api_key_auth_obj", + new_callable=AsyncMock, + return_value=valid_token, + ), + patch( + "litellm.proxy.auth.user_api_key_auth._hoist_request_destinations", + new_callable=AsyncMock, + ) as mock_hoist, + patch( + "litellm.proxy.auth.user_api_key_auth._enforce_key_and_fallback_model_access", + new_callable=AsyncMock, + side_effect=_assert_hoisted_first, + ) as mock_enforce, + ): + result = await _run_builder_with_key_lookup(get_key_object) + + assert result is valid_token + mock_hoist.assert_awaited_once() + mock_enforce.assert_awaited_once() + + def _mint_cli_session_token(monkeypatch, *, user_id="cli-admin"): """Mint a CLI session token for a PROXY_ADMIN user so auth resolves on the admin early-return path (no prisma/common_checks needed).""" @@ -3987,7 +4018,9 @@ async def test_non_admin_cli_session_token_reaches_production_auth_path(monkeypa assert call_kwargs["valid_token_dict"]["user_id"] == "internal-user-1" assert call_kwargs["valid_token_dict"]["team_id"] == "team-abc" assert call_kwargs["valid_token_dict"]["is_session_token"] is True - assert call_kwargs["valid_token_dict"]["user_role"] == LitellmUserRoles.INTERNAL_USER + assert ( + call_kwargs["valid_token_dict"]["user_role"] == LitellmUserRoles.INTERNAL_USER + ) assert result.is_session_token is True diff --git a/tests/test_litellm/proxy/credential_endpoints/test_access_decision.py b/tests/test_litellm/proxy/credential_endpoints/test_access_decision.py new file mode 100644 index 00000000000..295efb8f512 --- /dev/null +++ b/tests/test_litellm/proxy/credential_endpoints/test_access_decision.py @@ -0,0 +1,314 @@ +"""Exhaustive tests for the pure access-decision function. + +Each test names one specific reason a team-admin patch should be denied (or +allowed). Together they pin the security contract: changing this code with +the tests in place should fail a case named after what you broke. +""" + +from __future__ import annotations + +import pytest + +from litellm.models.credentials import CredentialInfo +from litellm.proxy.credential_endpoints.access_decision import ( + Allow, + Deny, + decide_credential_patch, +) + +_EXISTING_INFO = { + "credential_type": "logging", + "description": "tenant Langfuse", + "host": "https://cloud.langfuse.com", + "access": {"teams": ["team-A", "team-B"], "orgs": ["org-1"], "global": False}, +} + + +def _info(value): + return None if value is None else CredentialInfo.model_validate(value) + + +def _decision( + *, + is_proxy_admin: bool = False, + caller_team_admin_ids: frozenset[str] = frozenset({"team-T"}), + existing_info=_EXISTING_INFO, + patch_info=None, + patch_values=None, + patch_name_changed: bool = False, +): + return decide_credential_patch( + is_proxy_admin=is_proxy_admin, + caller_team_admin_ids=caller_team_admin_ids, + existing_info=_info(existing_info), + patch_info=_info(patch_info), + patch_values=patch_values, + patch_name_changed=patch_name_changed, + ) + + +class TestProxyAdminAllow: + def test_proxy_admin_allowed_on_value_change(self): + d = _decision( + is_proxy_admin=True, + patch_values={"api_key": "rotated"}, + patch_info={"credential_type": "logging"}, + ) + assert isinstance(d, Allow) + + def test_proxy_admin_allowed_on_global_flip(self): + d = _decision( + is_proxy_admin=True, + patch_info={"access": {"global": True}}, + ) + assert isinstance(d, Allow) + + def test_proxy_admin_allowed_on_rename(self): + d = _decision(is_proxy_admin=True, patch_name_changed=True) + assert isinstance(d, Allow) + + +class TestTeamAdminAllow: + def test_appending_own_team_id(self): + d = _decision( + patch_info={ + "access": {"teams": ["team-A", "team-B", "team-T"]}, + }, + ) + assert isinstance(d, Allow) + + def test_appending_only_own_team_id_when_no_prior_teams(self): + existing = {**_EXISTING_INFO, "access": {"global": False}} + d = _decision( + existing_info=existing, + patch_info={"access": {"teams": ["team-T"]}}, + ) + assert isinstance(d, Allow) + + def test_idempotent_when_already_granted(self): + existing = {**_EXISTING_INFO, "access": {"teams": ["team-T"]}} + d = _decision( + existing_info=existing, + patch_info={"access": {"teams": ["team-T"]}}, + ) + assert isinstance(d, Allow) + + +class TestTeamAdminDeny: + def test_not_team_admin_anywhere(self): + d = _decision( + caller_team_admin_ids=frozenset(), + patch_info={"access": {"teams": ["team-T"]}}, + ) + assert isinstance(d, Deny) + assert "proxy admin" in d.reason + + def test_rename(self): + d = _decision( + patch_name_changed=True, + patch_info={"access": {"teams": ["team-T"]}}, + ) + assert isinstance(d, Deny) + assert "credential_name" in d.reason + + def test_changing_credential_values(self): + d = _decision( + patch_values={"api_key": "stolen"}, + patch_info={"access": {"teams": ["team-T"]}}, + ) + assert isinstance(d, Deny) + assert "credential_values" in d.reason + + def test_empty_patch(self): + d = _decision(patch_info=None) + assert isinstance(d, Deny) + + @pytest.mark.parametrize( + "field", ["credential_type", "description", "host", "endpoint"] + ) + def test_changing_immutable_info_field(self, field): + d = _decision(patch_info={field: "x", "access": {"teams": ["team-T"]}}) + assert isinstance(d, Deny) + assert field in d.reason + + def test_patch_info_with_unknown_keys(self): + d = _decision(patch_info={"weird_field": 1}) + assert isinstance(d, Deny) + assert "weird_field" in d.reason + + def test_flipping_global(self): + d = _decision(patch_info={"access": {"global": True}}) + assert isinstance(d, Deny) + assert "global" in d.reason + + def test_editing_orgs(self): + d = _decision(patch_info={"access": {"orgs": ["org-new"]}}) + assert isinstance(d, Deny) + assert "orgs" in d.reason + + def test_no_op_resend_of_existing_global_is_allowed(self): + """The UI's Edit-access modal always sends the FULL access object + (so unchecking a team revokes). A non-admin re-sending the existing + global=false alongside their team patch must NOT be rejected as if + they were trying to flip the toggle. + + Before this fix the decider checked only "is global_ in the patch?" + which broke every UI save that included the unchanged global toggle + plus a team edit. + """ + d = _decision( + patch_info={ + "access": { + "global": False, + "teams": ["team-A", "team-B", "team-T"], + "orgs": ["org-1"], + } + } + ) + assert isinstance(d, Allow) + + def test_no_op_resend_of_existing_orgs_is_allowed(self): + """Same shape as global: a patch that includes the unchanged orgs + list alongside a team edit must pass.""" + d = _decision( + patch_info={ + "access": { + "global": False, + "teams": ["team-A", "team-B", "team-T"], + "orgs": ["org-1"], + } + } + ) + assert isinstance(d, Allow) + + def test_attempt_to_flip_global_when_existing_is_false(self): + """Direct flip from stored False to True still rejected.""" + d = _decision( + patch_info={"access": {"global": True, "teams": ["team-A", "team-B"]}} + ) + assert isinstance(d, Deny) + assert "global" in d.reason + + def test_attempt_to_flip_global_when_existing_is_true(self): + """Direct flip from stored True to False still rejected (it's still + a global mutation; only proxy-admin can change destination-wide reach).""" + existing = { + **_EXISTING_INFO, + "access": {**_EXISTING_INFO["access"], "global": True}, + } + d = _decision( + existing_info=existing, + patch_info={"access": {"global": False, "teams": ["team-A", "team-B"]}}, + ) + assert isinstance(d, Deny) + assert "global" in d.reason + + def test_attempt_to_change_orgs_is_rejected(self): + """Adding an org_id different from stored is still rejected.""" + d = _decision( + patch_info={ + "access": {"orgs": ["org-1", "org-2"], "teams": ["team-A", "team-B"]} + } + ) + assert isinstance(d, Deny) + assert "orgs" in d.reason + + def test_adding_foreign_team_id(self): + """foreign team_ids in the patch ARE caller input -- safe to echo.""" + d = _decision( + patch_info={ + "access": {"teams": ["team-A", "team-B", "team-foreign"]}, + }, + ) + assert isinstance(d, Deny) + assert d.from_user_input is True + assert "team-foreign" in d.reason + + def test_removing_foreign_team_grant(self): + """team-admin may NOT remove a team they don't admin. + + Reason intentionally does NOT echo the stored team_id (it's not + caller-typed; surfacing it would leak access list membership). + """ + d = _decision( + patch_info={ + "access": {"teams": ["team-A", "team-T"]}, + }, + ) + assert isinstance(d, Deny) + assert d.from_user_input is False + assert "team-B" not in d.reason + assert "may only revoke" in d.reason + + def test_replacing_teams_wholesale_with_foreign_remaining(self): + """Wholesale replacement that removes foreign grants is rejected. + + The stored team_ids that were dropped must not appear in the reason -- + they're access list contents, not caller input. + """ + d = _decision(patch_info={"access": {"teams": ["team-T"]}}) + assert isinstance(d, Deny) + assert d.from_user_input is False + assert "team-A" not in d.reason + assert "team-B" not in d.reason + + +class TestTeamAdminRevoke: + """A team-admin may revoke their OWN team's grant; never another's.""" + + def test_revoking_own_team_is_allowed(self): + existing = { + **_EXISTING_INFO, + "access": {"teams": ["team-A", "team-T"]}, + } + d = _decision( + existing_info=existing, + patch_info={"access": {"teams": ["team-A"]}}, + ) + assert isinstance(d, Allow) + + def test_revoking_own_team_when_only_grant(self): + """Saving an empty teams list when the caller was the sole grant.""" + existing = {**_EXISTING_INFO, "access": {"teams": ["team-T"]}} + d = _decision( + existing_info=existing, + patch_info={"access": {"teams": []}}, + ) + assert isinstance(d, Allow) + + def test_revoke_attempt_on_foreign_team_denied(self): + """A patch that removes a foreign team is still rejected, even if + the caller is also revoking their own. The foreign team_id MUST + NOT appear in the reason (stored access list content).""" + existing = { + **_EXISTING_INFO, + "access": {"teams": ["team-A", "team-B", "team-T"]}, + } + d = _decision( + existing_info=existing, + patch_info={"access": {"teams": ["team-A"]}}, # drops team-B AND team-T + ) + assert isinstance(d, Deny) + assert d.from_user_input is False + assert "team-B" not in d.reason + + +class TestTeamAdminMultipleTeams: + def test_can_add_multiple_own_team_ids(self): + d = _decision( + caller_team_admin_ids=frozenset({"team-T1", "team-T2"}), + patch_info={ + "access": {"teams": ["team-A", "team-B", "team-T1", "team-T2"]}, + }, + ) + assert isinstance(d, Allow) + + def test_one_own_one_foreign_is_deny(self): + d = _decision( + caller_team_admin_ids=frozenset({"team-T1"}), + patch_info={ + "access": {"teams": ["team-A", "team-B", "team-T1", "team-T2"]}, + }, + ) + assert isinstance(d, Deny) + assert "team-T2" in d.reason diff --git a/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py new file mode 100644 index 00000000000..25899bde7c4 --- /dev/null +++ b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py @@ -0,0 +1,764 @@ +"""Admin-gating on credential mutations. + +POST and DELETE on ``/credentials`` are proxy-admin only across the board +(both logging and provider credentials). The route gate was widened so +team-admins can PATCH ``access.teams`` on existing logging destinations, +which requires also reaching the GET endpoint — but creation and deletion +of any credential remain admin-only to keep platform infrastructure under +the platform admin's control. PATCH is widened for logging credentials only +via the pure ``decide_credential_patch`` decider tested separately. +""" + +import os +import sys + +import pytest +from fastapi import HTTPException +from unittest.mock import AsyncMock, MagicMock + +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +import litellm.proxy.credential_endpoints.endpoints as endpoints +from litellm.models.credentials import CredentialItem, UpdateCredentialItem +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.credential_endpoints.access_decision import OPAQUE_DENY_REASON +from litellm.types.utils import CreateCredentialItem + + +def _admin(): + return UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.PROXY_ADMIN) + + +def _member(): + return UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.INTERNAL_USER) + + +_LOGGING_INFO = {"credential_type": "logging", "description": "langfuse_otel"} + + +@pytest.fixture +def _connected_db(monkeypatch): + """A working prisma_client + repository so an allowed caller reaches success.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key") + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "llm_router", None) + repo = MagicMock() + repo.create = AsyncMock() + repo.delete_by_name = AsyncMock() + monkeypatch.setattr(endpoints, "CredentialsRepository", lambda _client: repo) + monkeypatch.setattr( + endpoints.CredentialAccessor, "upsert_credentials", lambda creds: None + ) + return repo + + +@pytest.mark.asyncio +async def test_create_logging_credential_forbidden_for_non_admin(_connected_db): + with pytest.raises(HTTPException) as exc: + await endpoints.create_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CreateCredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "h"}, + credential_info=_LOGGING_INFO, + ), + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + _connected_db.create.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_create_logging_credential_allowed_for_admin(_connected_db): + result = await endpoints.create_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CreateCredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "h"}, + credential_info=_LOGGING_INFO, + ), + user_api_key_dict=_admin(), + ) + assert result["success"] is True + _connected_db.create.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_create_provider_credential_forbidden_for_non_admin(_connected_db): + """POST is proxy-admin only even for provider credentials. + + The route gate now lets non-admins reach the credentials path so they can + PATCH ``access.teams`` on logging destinations they should be able to + self-assign. POST remains admin-only to keep credential creation a + platform-admin concern. + """ + with pytest.raises(HTTPException) as exc: + await endpoints.create_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CreateCredentialItem( + credential_name="openai", + credential_values={"api_key": "sk"}, + credential_info={"custom_llm_provider": "openai"}, + ), + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + _connected_db.create.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_update_logging_credential_forbidden_for_non_admin(_connected_db): + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={"access": {"global": True}}, + ), + credential_name="dest", + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_update_existing_logging_credential_forbidden_even_without_logging_patch( + _connected_db, monkeypatch +): + """A non-admin cannot edit a stored logging credential's values, even with a patch + that omits credential_info (the gate consults the in-memory credential too).""" + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "h"}, + credential_info=_LOGGING_INFO, + ) + ], + ) + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "evil"}, + credential_info={}, + ), + credential_name="dest", + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + + +def test_update_db_credential_preserves_existing_info_on_partial_patch(): + """A partial credential_info patch (e.g. only access from the Edit-access modal) must + merge into the stored info, not replace it -- otherwise the logging tag is dropped and + the destination vanishes from the registry after the next reload.""" + from litellm.proxy.credential_endpoints.endpoints import update_db_credential + + db = CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={ + "credential_type": "logging", + "description": "langfuse_otel", + "host": "h", + }, + ) + patch = CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={"access": {"global": True}}, + ) + + merged = update_db_credential(db, patch) + + assert merged.credential_info == { + "credential_type": "logging", + "description": "langfuse_otel", + "host": "h", + "access": {"global": True}, + } + + +def test_update_db_credential_preserves_untouched_access_subfields(): + """Veria regression: an access patch carrying only `teams` must NOT clobber + existing `global` / `orgs`. Pre-fix this caused a scope-tampering bug — + the decider only allowed team-admin patches that touched access.teams, + but the merge replaced the entire access object, silently dropping + access.global=true and any access.orgs entries. + """ + from litellm.proxy.credential_endpoints.endpoints import update_db_credential + + db = CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={ + "credential_type": "logging", + "description": "langfuse_otel", + "host": "h", + "access": { + "global": True, + "orgs": ["org-1", "org-2"], + "teams": ["team-A"], + }, + }, + ) + # Team-admin's allowed shape: add their team to access.teams. Crucially, + # they don't (and per the decider can't) include global/orgs. + patch = CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={"access": {"teams": ["team-A", "team-T"]}}, + ) + + merged = update_db_credential(db, patch) + + # access.global and access.orgs survive untouched; access.teams is updated. + assert merged.credential_info["access"] == { + "global": True, + "orgs": ["org-1", "org-2"], + "teams": ["team-A", "team-T"], + } + + +@pytest.mark.asyncio +async def test_delete_logging_credential_forbidden_for_non_admin( + _connected_db, monkeypatch +): + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="dest", + credential_values={}, + credential_info=_LOGGING_INFO, + ) + ], + ) + with pytest.raises(HTTPException) as exc: + await endpoints.delete_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential_name="dest", + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + _connected_db.delete_by_name.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_update_db_only_logging_credential_forbidden_for_non_admin( + _connected_db, monkeypatch +): + """A logging credential that exists ONLY in the DB (not resident in the + in-memory ``credential_list`` -- e.g. created on another scaled instance or + before a restart) must still gate a non-admin update. The gate falls back to + the DB so a credential_values-only patch can't redirect a logging + destination's endpoint without the proxy-admin check.""" + monkeypatch.setattr(litellm, "credential_list", []) # nothing in memory + _connected_db.find_by_name = AsyncMock( + return_value=CredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "h"}, + credential_info=_LOGGING_INFO, + ) + ) + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "evil"}, + credential_info={}, + ), + credential_name="dest", + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_delete_db_only_logging_credential_forbidden_for_non_admin( + _connected_db, monkeypatch +): + """Same DB-only fallback for delete: a non-admin can't delete a logging + credential that is resident only in the DB.""" + monkeypatch.setattr(litellm, "credential_list", []) + _connected_db.find_by_name = AsyncMock( + return_value=CredentialItem( + credential_name="dest", + credential_values={}, + credential_info=_LOGGING_INFO, + ) + ) + with pytest.raises(HTTPException) as exc: + await endpoints.delete_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential_name="dest", + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + _connected_db.delete_by_name.assert_not_awaited() + + +# --- team-admin self-assign tests (LIT-3850 follow-up) ---------------------- + +_DEST_WITH_TEAMS = { + "credential_type": "logging", + "description": "tenant Langfuse", + "host": "https://cloud.langfuse.com", + "access": {"teams": ["team-existing"]}, +} + + +def _team_admin_of(team_ids): + """A non-admin caller whose user_id is admin of the named teams. + + Combined with ``_patch_team_admin_lookup`` it mimics the real + ``_caller_grantable_team_ids`` resolution without touching the DB. + """ + return UserAPIKeyAuth( + api_key="k", user_role=LitellmUserRoles.INTERNAL_USER, user_id="ta-demo" + ) + + +@pytest.fixture +def _patch_team_admin_lookup(monkeypatch): + """Substitute the DB-backed grant-capability lookup with a configurable mock.""" + + holder = {"ids": frozenset()} + + async def _fake(user_api_key_dict, prisma_client): + return holder["ids"] + + monkeypatch.setattr(endpoints, "_caller_grantable_team_ids", _fake) + return holder + + +def _resident_logging_dest(): + return CredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "h"}, + credential_info=_DEST_WITH_TEAMS, + ) + + +@pytest.mark.asyncio +async def test_team_admin_can_append_own_team_to_access( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + monkeypatch.setattr(litellm, "credential_list", [_resident_logging_dest()]) + _connected_db.find_by_name = AsyncMock(return_value=_resident_logging_dest()) + _connected_db.update_by_name = AsyncMock() + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + + result = await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={"access": {"teams": ["team-existing", "team-T"]}}, + ), + credential_name="dest", + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert result["success"] is True + _connected_db.update_by_name.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_provider_credential_patch_forbidden_for_non_admin( + _connected_db, monkeypatch +): + """A team-admin (or any non-admin) cannot PATCH a non-logging credential. + + The route gate was widened to let team-admins reach /credentials/{name} + for logging-credential access edits. A provider credential is not + is_admin_gated_credential_info, so the decider block is skipped; without + an explicit else-branch a team-admin could rotate the upstream api_key. + """ + provider_cred = CredentialItem( + credential_name="openai-prod", + credential_values={"api_key": "sk-real"}, + credential_info={"custom_llm_provider": "openai"}, + ) + monkeypatch.setattr(litellm, "credential_list", [provider_cred]) + _connected_db.find_by_name = AsyncMock(return_value=provider_cred) + _connected_db.update_by_name = AsyncMock() + + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="openai-prod", + credential_values={"api_key": "sk-stolen"}, + credential_info={}, + ), + credential_name="openai-prod", + user_api_key_dict=_member(), + ) + assert exc.value.status_code == 403 + _connected_db.update_by_name.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_provider_credential_access_patch_bypass_forbidden( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + """Cursor BugBot regression: a team-admin can't sneak `access.teams` onto + a PROVIDER credential to route through the decider instead of the admin + gate. + + `is_admin_gated_credential_info(patch)` returns True for any patch + containing an `access` field, so previously a team-admin could PATCH a + provider credential with `{credential_info: {access: {teams: [...]}}}` + and reach `decide_credential_patch`, which would Allow because the patch + is just "add own team to access.teams". Gate must look at the STORED + credential's type, not the patch body. + """ + provider_cred = CredentialItem( + credential_name="openai-prod", + credential_values={"api_key": "sk-real"}, + credential_info={"custom_llm_provider": "openai"}, + ) + monkeypatch.setattr(litellm, "credential_list", [provider_cred]) + _connected_db.find_by_name = AsyncMock(return_value=provider_cred) + _connected_db.update_by_name = AsyncMock() + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=UpdateCredentialItem( + credential_info={"access": {"teams": ["team-T"]}}, + ), + credential_name="openai-prod", + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert exc.value.status_code == 403 + _connected_db.update_by_name.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_team_admin_can_revoke_own_team_grant( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + """A team-admin saving an access list without their own team_id revokes it.""" + existing = CredentialItem( + credential_name="dest", + credential_values={"langfuse_host": "h"}, + credential_info={ + "credential_type": "logging", + "description": "tenant Langfuse", + "host": "https://cloud.langfuse.com", + "access": {"teams": ["team-existing", "team-T"]}, + }, + ) + monkeypatch.setattr(litellm, "credential_list", [existing]) + _connected_db.find_by_name = AsyncMock(return_value=existing) + _connected_db.update_by_name = AsyncMock() + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + + result = await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={"access": {"teams": ["team-existing"]}}, + ), + credential_name="dest", + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert result["success"] is True + _connected_db.update_by_name.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_team_admin_cannot_grant_foreign_team( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + monkeypatch.setattr(litellm, "credential_list", [_resident_logging_dest()]) + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={ + "access": {"teams": ["team-existing", "team-foreign"]} + }, + ), + credential_name="dest", + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert exc.value.status_code == 403 + assert "team-foreign" in exc.value.detail["error"] + + +@pytest.mark.asyncio +async def test_team_admin_cannot_rotate_credential_values( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + monkeypatch.setattr(litellm, "credential_list", [_resident_logging_dest()]) + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={"public_key": "pk-stolen"}, + credential_info={"access": {"teams": ["team-existing", "team-T"]}}, + ), + credential_name="dest", + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert exc.value.status_code == 403 + # Endpoint collapses non-caller-input Deny reasons to the opaque message + # so PATCH /credentials/{name} can't be used as an existence oracle. + assert exc.value.detail["error"] == OPAQUE_DENY_REASON + + +@pytest.mark.asyncio +async def test_team_admin_cannot_flip_global( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + monkeypatch.setattr(litellm, "credential_list", [_resident_logging_dest()]) + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=CredentialItem( + credential_name="dest", + credential_values={}, + credential_info={"access": {"global": True}}, + ), + credential_name="dest", + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert exc.value.status_code == 403 + assert exc.value.detail["error"] == OPAQUE_DENY_REASON + + +@pytest.mark.asyncio +async def test_get_credentials_filters_to_logging_for_non_admin( + monkeypatch, _patch_team_admin_lookup +): + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="openai", + credential_values={"api_key": "sk-secret"}, + credential_info={"custom_llm_provider": "openai"}, + ), + CredentialItem( + credential_name="poc-langfuse", + credential_values={"public_key": "pk-1"}, + credential_info=_DEST_WITH_TEAMS, + ), + ], + ) + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + response = await endpoints.get_credentials( + request=MagicMock(), + fastapi_response=MagicMock(), + user_api_key_dict=_team_admin_of(["team-T"]), + ) + names = [c["credential_name"] for c in response["credentials"]] + assert names == ["poc-langfuse"] + + +@pytest.mark.asyncio +async def test_get_credentials_forbidden_for_plain_user( + monkeypatch, _patch_team_admin_lookup +): + """Veria F2 regression: a plain internal_user (no team-admin or + org-admin status anywhere) gets 403, NOT a filtered list. The previous + handler returned destination names, hosts, and scope metadata to any + authenticated caller because the route gate was widened to support + team-admin self-assignment. + """ + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="poc-langfuse", + credential_values={"public_key": "pk-1"}, + credential_info=_DEST_WITH_TEAMS, + ), + ], + ) + _patch_team_admin_lookup["ids"] = frozenset() # admins nothing + + with pytest.raises(HTTPException) as exc: + await endpoints.get_credentials( + request=MagicMock(), + fastapi_response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + api_key="k", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="plain-user", + ), + ) + assert exc.value.status_code == 403 + assert "team-admin" in exc.value.detail["error"] + + +def test_patch_credentials_route_targets_update_credential(): + """Regression: the @router.patch decorator on /credentials/{name:path} must + decorate update_credential, not one of the extracted helpers. A misplaced + decorator landed once during the 7ecc1d49 split and the unit tests didn't + catch it because they import the handler function directly; this asserts + the FastAPI routing table actually points at update_credential. + """ + from fastapi.routing import APIRoute + + patch_route = next( + route + for route in endpoints.router.routes + if isinstance(route, APIRoute) + and route.path == "/credentials/{credential_name:path}" + and "PATCH" in route.methods + ) + assert patch_route.endpoint is endpoints.update_credential + + +@pytest.mark.asyncio +async def test_patch_credentials_does_not_leak_credential_type( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + """Existence-oracle regression: a team-admin probing a credential they don't own + must NOT be able to distinguish "logging credential, not yours" from + "provider credential" or "doesn't exist" by comparing 403 detail strings. + + Pre-fix the decider returned reasons like "access.global is proxy-admin only" + while `_require_proxy_admin` returned a fixed string, so the same probe + (e.g. PATCH `{credential_info: {access: {global: true}}}`) would yield + different bodies depending on the stored type. All three paths now return + the same opaque message. + """ + provider = CredentialItem( + credential_name="openai-prod", + credential_values={"api_key": "sk-real"}, + credential_info={"custom_llm_provider": "openai"}, + ) + logging_other = CredentialItem( + credential_name="other-langfuse", + credential_values={"langfuse_host": "h"}, + credential_info={ + "credential_type": "logging", + "description": "tenant Langfuse", + "host": "https://cloud.langfuse.com", + "access": {"teams": ["team-other"]}, + }, + ) + monkeypatch.setattr(litellm, "credential_list", [provider, logging_other]) + _connected_db.find_by_name = AsyncMock( + side_effect=lambda name: provider if name == "openai-prod" else logging_other + ) + # Caller is team-admin of team-T (NOT team-other, so can't legitimately + # edit logging_other either), probing with a patch that touches a + # decider-protected field to maximally expose any branch divergence. + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + probe_patch = UpdateCredentialItem( + credential_info={"access": {"global": True}}, + ) + + bodies: list[object] = [] + for name in ("openai-prod", "other-langfuse", "does-not-exist"): + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=probe_patch, + credential_name=name, + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert exc.value.status_code == 403 + bodies.append(exc.value.detail) + + assert bodies[0] == bodies[1] == bodies[2] == {"error": OPAQUE_DENY_REASON} + + +@pytest.mark.asyncio +async def test_patch_credentials_echoes_foreign_team_id_to_legit_team_admin( + _connected_db, _patch_team_admin_lookup, monkeypatch +): + """The one accepted leak: when a team-admin tries to grant a team_id they + typed in the patch and don't admin, the response names that team_id so + the UI can render a useful error. The team_id was caller input, so it + isn't an existence oracle (the caller already knew the value). + """ + monkeypatch.setattr(litellm, "credential_list", [_resident_logging_dest()]) + _connected_db.find_by_name = AsyncMock(return_value=_resident_logging_dest()) + _patch_team_admin_lookup["ids"] = frozenset({"team-T"}) + + with pytest.raises(HTTPException) as exc: + await endpoints.update_credential( + request=MagicMock(), + fastapi_response=MagicMock(), + credential=UpdateCredentialItem( + credential_info={ + "access": {"teams": ["team-existing", "team-foreign"]} + }, + ), + credential_name="dest", + user_api_key_dict=_team_admin_of(["team-T"]), + ) + assert exc.value.status_code == 403 + assert "team-foreign" in exc.value.detail["error"] + + +@pytest.mark.asyncio +async def test_get_credentials_returns_all_for_proxy_admin(monkeypatch): + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="openai", + credential_values={"api_key": "sk-secret"}, + credential_info={"custom_llm_provider": "openai"}, + ), + CredentialItem( + credential_name="poc-langfuse", + credential_values={"public_key": "pk-1"}, + credential_info=_DEST_WITH_TEAMS, + ), + ], + ) + response = await endpoints.get_credentials( + request=MagicMock(), + fastapi_response=MagicMock(), + user_api_key_dict=_admin(), + ) + names = sorted(c["credential_name"] for c in response["credentials"]) + assert names == ["openai", "poc-langfuse"] diff --git a/tests/test_litellm/proxy/management_endpoints/test_logging_exporter_validation.py b/tests/test_litellm/proxy/management_endpoints/test_logging_exporter_validation.py new file mode 100644 index 00000000000..1a12b5bf7a2 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_logging_exporter_validation.py @@ -0,0 +1,385 @@ +"""Validation for admin-owned logging-exporter assignment on key/team/org. + +The single ``validate_logging_exporter_assignment`` runs across all four +endpoints (``/team/new``, ``/team/update``, ``/key/generate``, ``/key/update``, +``/organization/*``); each call site computes the relevant +``caller_is_team_admin`` / ``caller_is_org_admin`` flags from the loaded +team or org and passes them in. Proxy admin always passes. +""" + +import os +import sys + +import pytest +from fastapi import HTTPException + +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +from litellm.models.credentials import CredentialItem +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.management_endpoints.logging_exporter_validation import ( + is_admin_gated_credential_info, + validate_credential_access, + validate_logging_exporter_assignment, +) + + +@pytest.fixture +def _registry(): + original = litellm.credential_list + litellm.credential_list = [ + # global: visible to (and assignable by) every scope. + CredentialItem( + credential_name="langfuse-eu", + credential_values={}, + credential_info={ + "credential_type": "logging", + "description": "langfuse_otel", + "access": {"global": True}, + }, + ), + # scoped to one team / one org: assignable only within that scope. + CredentialItem( + credential_name="arize-ds", + credential_values={}, + credential_info={ + "credential_type": "logging", + "description": "arize", + "access": {"teams": ["ds-team"], "orgs": ["ds-org"]}, + }, + ), + # explicit global/default: assignable by anyone via the auto_enable escape. + CredentialItem( + credential_name="central-default", + credential_values={}, + credential_info={ + "credential_type": "logging", + "description": "arize", + "auto_enable": True, + }, + ), + CredentialItem( + credential_name="openai-key", + credential_values={}, + credential_info={"custom_llm_provider": "openai"}, # provider credential + ), + ] + try: + yield + finally: + litellm.credential_list = original + + +def _admin(): + return UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.PROXY_ADMIN) + + +def _non_admin(): + return UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.INTERNAL_USER) + + +def _ok(metadata): + return {"logging_exporters": metadata} + + +# --- Role allow paths ------------------------------------------------------- + + +def test_proxy_admin_always_allowed(_registry): + """No flags needed; proxy_admin role suffices.""" + validate_logging_exporter_assignment(_ok(["langfuse-eu"]), _admin()) + + +def test_team_admin_flag_allows_non_admin(_registry): + """A non-admin caller flagged caller_is_team_admin=True passes.""" + validate_logging_exporter_assignment( + _ok(["langfuse-eu"]), + _non_admin(), + caller_is_team_admin=True, + ) + + +def test_org_admin_flag_allows_non_admin(_registry): + """A non-admin caller flagged caller_is_org_admin=True passes.""" + validate_logging_exporter_assignment( + _ok(["langfuse-eu"]), + _non_admin(), + caller_is_org_admin=True, + ) + + +def test_both_flags_set_allows_non_admin(_registry): + """Setting both flags is fine; they're independent OR-ed allows.""" + validate_logging_exporter_assignment( + _ok(["langfuse-eu"]), + _non_admin(), + caller_is_team_admin=True, + caller_is_org_admin=True, + ) + + +def test_non_admin_with_no_flags_is_forbidden(_registry): + """The headline deny: internal_user with no team/org admin context.""" + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment(_ok(["langfuse-eu"]), _non_admin()) + assert exc.value.status_code == 403 + + +def test_proxy_admin_overrides_falsy_flags(_registry): + """proxy_admin role wins even when both flags are False.""" + validate_logging_exporter_assignment( + _ok(["langfuse-eu"]), + _admin(), + caller_is_team_admin=False, + caller_is_org_admin=False, + ) + + +# --- Scope checks: a non-proxy-admin may only name destinations granted to them - + + +def test_team_admin_can_assign_destination_granted_to_their_team(_registry): + """arize-ds is granted to ds-team; a team admin writing in ds-team's scope may + name it.""" + validate_logging_exporter_assignment( + _ok(["arize-ds"]), + _non_admin(), + caller_is_team_admin=True, + scope_team_id="ds-team", + ) + + +def test_team_admin_cannot_assign_destination_not_granted_to_their_team(_registry): + """The headline leak: a team admin of another team names ds-team's destination. + Pre-fix this passed (only the name was checked); now it is a 403.""" + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment( + _ok(["arize-ds"]), + _non_admin(), + caller_is_team_admin=True, + scope_team_id="platform-team", + ) + assert exc.value.status_code == 403 + + +def test_org_admin_can_assign_destination_granted_to_their_org(_registry): + validate_logging_exporter_assignment( + _ok(["arize-ds"]), + _non_admin(), + caller_is_org_admin=True, + scope_org_id="ds-org", + ) + + +def test_org_admin_cannot_assign_destination_not_granted_to_their_org(_registry): + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment( + _ok(["arize-ds"]), + _non_admin(), + caller_is_org_admin=True, + scope_org_id="other-org", + ) + assert exc.value.status_code == 403 + + +def test_proxy_admin_can_assign_any_destination(_registry): + """Proxy admin skips the scope check entirely; arize-ds is granted to no scope + the admin is in, yet the write is allowed.""" + validate_logging_exporter_assignment( + _ok(["arize-ds"]), + _admin(), + scope_team_id="platform-team", + ) + + +def test_team_admin_can_assign_auto_enable_default(_registry): + """An explicit global/default (auto_enable) is assignable by any admin scope, + the way a global destination is.""" + validate_logging_exporter_assignment( + _ok(["central-default"]), + _non_admin(), + caller_is_team_admin=True, + scope_team_id="platform-team", + ) + + +def test_team_admin_can_assign_global_destination(_registry): + """access.global makes a destination visible to every scope, so a team admin + in any team may name it.""" + validate_logging_exporter_assignment( + _ok(["langfuse-eu"]), + _non_admin(), + caller_is_team_admin=True, + scope_team_id="platform-team", + ) + + +# --- Shape / registry checks (run regardless of who's calling) -------------- + + +def test_unknown_credential_rejected_for_admin(_registry): + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment(_ok(["does-not-exist"]), _admin()) + assert exc.value.status_code == 400 + + +def test_unknown_credential_rejected_for_team_admin(_registry): + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment( + _ok(["does-not-exist"]), + _non_admin(), + caller_is_team_admin=True, + ) + assert exc.value.status_code == 400 + + +def test_provider_credential_rejected(_registry): + """openai-key exists but is provider-typed, not a logging destination.""" + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment(_ok(["openai-key"]), _admin()) + assert exc.value.status_code == 400 + + +def test_non_list_is_rejected(_registry): + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment( + {"logging_exporters": "langfuse-eu"}, _admin() + ) + assert exc.value.status_code == 400 + + +def test_noop_when_field_absent(_registry): + """An update that does not touch logging_exporters skips the gate even + for a non-admin with no flags.""" + validate_logging_exporter_assignment({"some_other_key": 1}, _non_admin()) + validate_logging_exporter_assignment(None, _non_admin()) + + +# --- Veria F4: removal-via-omission ---------------------------------------- +# +# Update endpoints replace stored metadata wholesale. A caller can wipe an +# admin-assigned `logging_exporters` by sending a `metadata` payload that +# omits the field. The validator must catch this when ``existing_metadata`` +# is passed. + + +def test_removal_via_omission_blocked_for_non_admin(_registry): + """A non-admin with no flags cannot wipe an admin-assigned exporter by + submitting metadata without logging_exporters.""" + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment( + {"some_other_key": 1}, # no logging_exporters in the new payload + _non_admin(), + existing_metadata={"logging_exporters": ["langfuse-eu"]}, + ) + assert exc.value.status_code == 403 + + +def test_removal_via_omission_allowed_for_proxy_admin(_registry): + """Proxy admin may drop the exporter via omission.""" + validate_logging_exporter_assignment( + {"some_other_key": 1}, + _admin(), + existing_metadata={"logging_exporters": ["langfuse-eu"]}, + ) + + +def test_removal_via_omission_allowed_for_team_admin(_registry): + """A team-admin of the owning team may drop the exporter.""" + validate_logging_exporter_assignment( + {"some_other_key": 1}, + _non_admin(), + caller_is_team_admin=True, + existing_metadata={"logging_exporters": ["langfuse-eu"]}, + ) + + +def test_explicit_empty_list_blocked_for_non_admin(_registry): + """A non-admin submitting `logging_exporters: []` over a non-empty stored + value is a removal write and must be gated.""" + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment( + {"logging_exporters": []}, + _non_admin(), + existing_metadata={"logging_exporters": ["langfuse-eu"]}, + ) + assert exc.value.status_code == 403 + + +def test_explicit_null_blocked_for_non_admin(_registry): + """`logging_exporters: null` over a non-empty stored value is also a + removal; the validator's shape check would reject it as non-list, but + F4's authorization gate must fire first.""" + with pytest.raises(HTTPException) as exc: + validate_logging_exporter_assignment( + {"logging_exporters": None}, + _non_admin(), + existing_metadata={"logging_exporters": ["langfuse-eu"]}, + ) + assert exc.value.status_code == 403 + + +def test_unchanged_value_is_noop(_registry): + """A metadata payload that re-sends the SAME logging_exporters value is + a noop and skips the gate even for a non-admin -- there is no net change + to authorize.""" + validate_logging_exporter_assignment( + {"logging_exporters": ["langfuse-eu"]}, + _non_admin(), + existing_metadata={"logging_exporters": ["langfuse-eu"]}, + ) + + +def test_omitted_on_both_sides_is_noop(_registry): + """A metadata update that doesn't touch logging_exporters on a row that + never had one is a noop.""" + validate_logging_exporter_assignment( + {"some_other_key": 1}, + _non_admin(), + existing_metadata={"some_other_key": 0}, + ) + + +# --- is_admin_gated_credential_info / validate_credential_access ------------ + + +@pytest.mark.parametrize( + "credential_info, gated", + [ + ({"credential_type": "logging"}, True), + ({"access": {"global": True}}, True), + ({"credential_type": "logging", "access": {"teams": ["t"]}}, True), + ({"custom_llm_provider": "openai"}, False), + ({}, False), + (None, False), + ], +) +def test_is_admin_gated_credential_info(credential_info, gated): + assert is_admin_gated_credential_info(credential_info) is gated + + +def test_validate_credential_access_accepts_valid_object(): + validate_credential_access( + {"access": {"global": False, "teams": ["t1", "t2"], "orgs": ["o1"]}} + ) + + +def test_validate_credential_access_noop_without_access(): + validate_credential_access({"credential_type": "logging"}) + validate_credential_access(None) + + +@pytest.mark.parametrize( + "access", + [ + 5, # not an object + {"global": "yes"}, # global must be bool + {"teams": "t1"}, # teams must be a list + {"orgs": [1, 2]}, # orgs must be strings + ], +) +def test_validate_credential_access_rejects_bad_shape(access): + with pytest.raises(HTTPException) as exc: + validate_credential_access({"access": access}) + assert exc.value.status_code == 400 diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 74f681bb97e..e07e74bea9d 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -4798,3 +4798,416 @@ async def test_add_litellm_data_to_request_claude_code_drop_params( ) assert updated.get("drop_params") == expected_drop_params + + +@pytest.fixture +def _seeded_logging_credentials(): + from litellm.models.credentials import CredentialItem + + original = litellm.credential_list + litellm.credential_list = [ + CredentialItem( + credential_name="langfuse-eu", + credential_values={ + "langfuse_host": "https://cloud.langfuse.com", + "langfuse_public_key": "pk-eu", + "langfuse_secret_key": "sk-eu", + }, + credential_info={ + "credential_type": "logging", + "description": "langfuse_otel", + "access": {"global": True}, + }, + ), + CredentialItem( + credential_name="arize-prod", + credential_values={ + "arize_space_id": "S", + "arize_api_key": "K", + "arize_project_name": "tenant-arize", + }, + credential_info={ + "credential_type": "logging", + "description": "arize", + "access": {"global": True}, + }, + ), + # A provider credential that must never resolve as a logging destination. + CredentialItem( + credential_name="openai-key", + credential_values={"api_key": "sk-openai"}, + credential_info={"custom_llm_provider": "openai"}, + ), + ] + try: + yield + finally: + litellm.credential_list = original + + +def _auth(team_exporters=None, token=None, org_id=None, team_id=None): + return UserAPIKeyAuth( + api_key="hashed-key", + token=token, + org_id=org_id, + team_id=team_id, + team_metadata=({"logging_exporters": team_exporters} if team_exporters else {}), + ) + + +@pytest.mark.asyncio +async def test_resolve_logging_exporters_team_level(_seeded_logging_credentials): + # team_metadata is the team's own (not shadowed); resolves without a DB fetch. + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + destinations, backends = await _resolve_logging_exporters( + _auth(team_exporters=["langfuse-eu"]) + ) + assert {d["endpoint"] for d in destinations} == { + "https://cloud.langfuse.com/api/public/otel" + } + assert backends == ["langfuse_otel"] + + +@pytest.mark.asyncio +async def test_resolve_logging_exporters_unions_key_team_org( + _seeded_logging_credentials, monkeypatch +): + # key + org are read from their OWN records (the key's .metadata is team-shadowed), + # team from team_metadata. All three union, deduped. + from types import SimpleNamespace + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.auth import auth_checks + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr( + auth_checks, + "get_key_object", + AsyncMock( + return_value=SimpleNamespace(metadata={"logging_exporters": ["arize-prod"]}) + ), + ) + monkeypatch.setattr( + auth_checks, + "get_org_object", + AsyncMock( + return_value=SimpleNamespace( + metadata={"logging_exporters": ["langfuse-eu"]} + ) + ), + ) + + destinations, backends = await _resolve_logging_exporters( + _auth(team_exporters=["langfuse-eu"], token="hashed-key", org_id="org-1") + ) + + assert {d["endpoint"] for d in destinations} == { + "https://cloud.langfuse.com/api/public/otel", # team + org (deduped) + "https://otlp.arize.com/v1", # key + } + assert set(backends) == {"langfuse_otel", "arize"} + + +@pytest.mark.asyncio +async def test_resolve_logging_exporters_carries_arize_project( + _seeded_logging_credentials, +): + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + destinations, _ = await _resolve_logging_exporters( + _auth(team_exporters=["arize-prod"]) + ) + + assert destinations == [ + { + "callback_name": "arize", + "endpoint": "https://otlp.arize.com/v1", + "headers": {"space_id": "S", "api_key": "K"}, + "resource_attributes": { + "model_id": "tenant-arize", + "arize.project.name": "tenant-arize", + }, + } + ] + + +@pytest.mark.asyncio +async def test_resolve_logging_exporters_empty_without_assignment( + _seeded_logging_credentials, +): + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + destinations, backends = await _resolve_logging_exporters(_auth()) + assert destinations == [] and backends == [] + + +@pytest.mark.asyncio +async def test_resolve_logging_exporters_skips_unknown_and_provider_creds( + _seeded_logging_credentials, +): + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + # unknown name + a provider credential (not credential_type=logging) -> nothing + destinations, backends = await _resolve_logging_exporters( + _auth(team_exporters=["does-not-exist", "openai-key"]) + ) + assert destinations == [] and backends == [] + + +@pytest.mark.asyncio +async def test_apply_admin_logging_exporters_stamps_and_activates( + _seeded_logging_credentials, +): + from litellm.integrations.otel.plumbing.context import ( + _request_destinations, + request_destinations, + ) + from litellm.proxy.litellm_pre_call_utils import _apply_admin_logging_exporters + + token = _request_destinations.set(()) + data: dict = {} + try: + await _apply_admin_logging_exporters( + data, _auth(team_exporters=["langfuse-eu"]) + ) + + # destinations live under litellm_metadata so the body does not leak an + # unknown top-level key to the provider; the top-level key stays absent + assert "otel_destinations" not in data + destinations = data["litellm_metadata"]["otel_destinations"] + assert destinations[0]["callback_name"] == "langfuse_otel" + assert ( + destinations[0]["endpoint"] == "https://cloud.langfuse.com/api/public/otel" + ) + # the backend is activated for the request + assert "langfuse_otel" in data["success_callback"] + context_destinations = request_destinations() + assert len(context_destinations) == 1 + assert ( + context_destinations[0].endpoint + == "https://cloud.langfuse.com/api/public/otel" + ) + finally: + _request_destinations.reset(token) + + +@pytest.mark.asyncio +async def test_apply_admin_logging_exporters_registers_on_failure( + _seeded_logging_credentials, +): + """An admin-owned destination must capture a FAILED upstream call, not only a + successful one. + + Each resolved backend is a dynamic logging callback that fires per-request; + registering it on ``success_callback`` alone means a 401/timeout never reaches + the backend's failure callback, so the destination's trace lands with no + error gen-AI span. Both lists must carry every backend, deduped against any + pre-existing entry. + """ + from litellm.integrations.otel.plumbing.context import _request_destinations + from litellm.proxy.litellm_pre_call_utils import _apply_admin_logging_exporters + + token = _request_destinations.set(()) + # Seed a pre-existing failure callback to prove backends are unioned in, not + # overwriting, and that a duplicate backend is not appended twice. + data: dict = {"failure_callback": ["arize"]} + try: + await _apply_admin_logging_exporters( + data, _auth(team_exporters=["langfuse-eu", "arize-prod"]) + ) + for callback_list in ("success_callback", "failure_callback"): + registered = data[callback_list] + assert "langfuse_otel" in registered + assert "arize" in registered + assert registered.count("arize") == 1 + finally: + _request_destinations.reset(token) + + +_LANGFUSE_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" +_ARIZE_ENDPOINT = "https://otlp.arize.com/v1" + + +@pytest.fixture +def _seeded_logging_credentials_with_access(): + """``access`` is visibility, not enablement. ``langfuse-eu`` is granted to + ``team-eu``/``org-eu`` but never auto-fires; ``arize-global`` carries + ``access.global`` to prove global visibility alone STILL does not auto-fire; + ``arize-default`` is the explicit ``auto_enable`` global/default.""" + from litellm.models.credentials import CredentialItem + + original = litellm.credential_list + litellm.credential_list = [ + CredentialItem( + credential_name="langfuse-eu", + credential_values={ + "langfuse_host": "https://cloud.langfuse.com", + "langfuse_public_key": "pk-eu", + "langfuse_secret_key": "sk-eu", + }, + credential_info={ + "credential_type": "logging", + "description": "langfuse_otel", + "access": {"teams": ["team-eu"], "orgs": ["org-eu"]}, + }, + ), + CredentialItem( + credential_name="arize-global", + credential_values={"arize_space_id": "S", "arize_api_key": "K"}, + credential_info={ + "credential_type": "logging", + "description": "arize", + "access": {"global": True}, + }, + ), + CredentialItem( + credential_name="arize-default", + credential_values={"arize_space_id": "D", "arize_api_key": "K"}, + credential_info={ + "credential_type": "logging", + "description": "arize", + "auto_enable": True, + }, + ), + ] + try: + yield + finally: + litellm.credential_list = original + + +@pytest.mark.asyncio +async def test_resolve_grant_does_not_auto_enable( + _seeded_logging_credentials_with_access, +): + """Granting a destination to a team (or globally) must NOT enable it for the + team's requests. The pre-fix resolver fired on access alone; this pins that + access is now visibility-only. ``arize-default`` (auto_enable) is the only thing + that fires for an unassigned team-eu caller.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + destinations, _ = await _resolve_logging_exporters(_auth(team_id="team-eu")) + + # langfuse-eu is granted to team-eu and arize-global is access.global, yet + # neither fires because neither is named; only the explicit auto_enable does. + assert {d["endpoint"] for d in destinations} == {_ARIZE_ENDPOINT} + + +@pytest.mark.asyncio +async def test_resolve_name_with_grant_enables( + _seeded_logging_credentials_with_access, +): + """Naming a destination the caller is granted enables it (alongside the + auto_enable default).""" + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + destinations, _ = await _resolve_logging_exporters( + _auth(team_id="team-eu", team_exporters=["langfuse-eu"]) + ) + + assert {d["endpoint"] for d in destinations} == {_LANGFUSE_ENDPOINT, _ARIZE_ENDPOINT} + + +@pytest.mark.asyncio +async def test_resolve_name_without_visibility_is_dropped( + _seeded_logging_credentials_with_access, +): + """A name that points at a destination NOT visible to the request identity is + defensively ignored, so a stale or cross-tenant assignment can never route + traffic out. team-other names langfuse-eu (granted only to team-eu).""" + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + destinations, _ = await _resolve_logging_exporters( + _auth(team_id="team-other", team_exporters=["langfuse-eu"]) + ) + + # langfuse-eu dropped (not visible to team-other); only auto_enable survives. + assert {d["endpoint"] for d in destinations} == {_ARIZE_ENDPOINT} + + +@pytest.mark.asyncio +async def test_resolve_access_global_alone_does_not_fire( + _seeded_logging_credentials_with_access, +): + """The headline regression: a destination with access.global but no auto_enable + and no name must NOT fire for an unassigned caller. Mutating the resolver back to + selecting on access alone re-adds arize-global here and fails this test.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + # an org with no grants, no names: only the auto_enable default fires. + destinations, _ = await _resolve_logging_exporters(_auth(org_id="org-unrelated")) + + # exactly one destination -- the auto_enable arize-default (space_id "D"). + # arize-global shares the arize endpoint but carries space_id "S"; if access.global + # auto-fired it would survive as a SECOND destination here. + assert len(destinations) == 1 + assert destinations[0]["endpoint"] == _ARIZE_ENDPOINT + assert destinations[0]["headers"]["space_id"] == "D" + + +@pytest.mark.asyncio +async def test_resolve_logging_exporters_access_default_deny( + _seeded_logging_credentials, +): + """With no auto_enable and no identity assignment, nothing resolves even though + the seeded creds are access.global-visible -- visibility never invents a + destination.""" + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + destinations, backends = await _resolve_logging_exporters( + _auth(team_id="team-eu", org_id="org-eu") + ) + assert destinations == [] and backends == [] + + +@pytest.mark.asyncio +async def test_resolve_org_scoped_via_team_when_token_has_no_org_id(monkeypatch): + """A team key whose token carries no org_id must still resolve an org-scoped + destination, via the team's organization_id. The write gate loads the team and + accepts the assignment, so the resolver must agree (M1); without the fallback the + org-granted destination is named but invisible (org_id None) and silently dropped. + Reverting _effective_org_id to user_api_key_dict.org_id fails this test.""" + from types import SimpleNamespace + + import litellm.proxy.proxy_server as proxy_server + from litellm.models.credentials import CredentialItem + from litellm.proxy.auth import auth_checks + from litellm.proxy.litellm_pre_call_utils import _resolve_logging_exporters + + original = litellm.credential_list + litellm.credential_list = [ + CredentialItem( + credential_name="arize-org", + credential_values={"arize_space_id": "S", "arize_api_key": "K"}, + credential_info={ + "credential_type": "logging", + "description": "arize", + "access": {"orgs": ["org-7"]}, + }, + ), + ] + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "user_api_key_cache", MagicMock()) + monkeypatch.setattr( + auth_checks, "get_key_object", AsyncMock(return_value=SimpleNamespace(metadata={})) + ) + monkeypatch.setattr( + auth_checks, + "get_org_object", + AsyncMock(return_value=SimpleNamespace(metadata={})), + ) + # the key's token has no org_id; the team it belongs to is in org-7. + monkeypatch.setattr( + auth_checks, + "get_team_object", + AsyncMock(return_value=SimpleNamespace(organization_id="org-7")), + ) + try: + destinations, _ = await _resolve_logging_exporters( + _auth(team_id="team-x", team_exporters=["arize-org"]) + ) + assert {d["endpoint"] for d in destinations} == {"https://otlp.arize.com/v1"} + finally: + litellm.credential_list = original diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 01c5a241562..19cd84b31d6 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,5 +1,5 @@ { - "@typescript-eslint/no-explicit-any": 2026, - "complexity": 128, - "max-depth": 61 + "@typescript-eslint/no-explicit-any": 2029, + "complexity": 129, + "max-depth": 62 } diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index be9015e3730..4db7b79c6b8 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -76,6 +76,7 @@ interface EditTeamModalProps { import { updateExistingKeys } from "@/utils/dataUtils"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; +import LoggingExportersSelect from "./logging_credentials/LoggingExportersSelect"; import { Member, teamCreateCall } from "./networking"; import { ModelSelect } from "./ModelSelect/ModelSelect"; @@ -469,6 +470,22 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser formValues.metadata = JSON.stringify(metadata); } + // Merge admin-owned logging_exporters into metadata at create time so the + // user can assign destinations from the new-team form (instead of create-then-edit). + if (Array.isArray(formValues.logging_exporters) && formValues.logging_exporters.length > 0) { + let metadata: Record = {}; + if (typeof formValues.metadata === "string" && formValues.metadata.trim().length > 0) { + try { + metadata = JSON.parse(formValues.metadata); + } catch (e) { + console.warn("Invalid JSON in metadata field, starting with empty object"); + } + } + metadata = { ...metadata, logging_exporters: formValues.logging_exporters }; + formValues.metadata = JSON.stringify(metadata); + } + delete formValues.logging_exporters; + if (formValues.secret_manager_settings) { if (typeof formValues.secret_manager_settings === "string") { if (formValues.secret_manager_settings.trim() === "") { @@ -1569,6 +1586,14 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser Logging Settings + + +
{ @@ -90,4 +90,93 @@ describe("LoggingCallbacksTable", () => { expect(getByText("Success")).toBeInTheDocument(); expect(getByText("Failure")).toBeInTheDocument(); }); + + const NO_VARS = { + SLACK_WEBHOOK_URL: null, + LANGFUSE_PUBLIC_KEY: null, + LANGFUSE_SECRET_KEY: null, + LANGFUSE_HOST: null, + OPENMETER_API_KEY: null, + }; + + it("renders a global destination's scope as a Global tag with no mode badge", () => { + const { getByText, queryByText } = render( + , + ); + expect(getByText("Global")).toBeInTheDocument(); + // a destination has no success/failure mode badge + expect(queryByText("Success")).not.toBeInTheDocument(); + }); + + it("renders a scoped destination's resolved teams and orgs as labeled tags", () => { + const { getByText } = render( + , + ); + expect(getByText("team: t1")).toBeInTheDocument(); + expect(getByText("team: t2")).toBeInTheDocument(); + expect(getByText("org: o1")).toBeInTheDocument(); + }); + + it("a destination row fires onEditAccess and onDelete, never onTest", () => { + const onEditAccess = vi.fn(); + const onDelete = vi.fn(); + const onTest = vi.fn(); + const { getByText } = render( + , + ); + const row = getByText("dest").closest("tr") as HTMLElement; + const scoped = within(row); + // destination rows expose edit-access + delete, and no test action + expect(scoped.queryByTestId("test-callback")).not.toBeInTheDocument(); + fireEvent.click(scoped.getByTestId("edit-access")); + fireEvent.click(scoped.getByTestId("delete-destination")); + expect(onEditAccess).toHaveBeenCalledWith(expect.objectContaining({ credentialName: "dest" })); + expect(onDelete).toHaveBeenCalledWith(expect.objectContaining({ credentialName: "dest" })); + expect(onTest).not.toHaveBeenCalled(); + }); + + it("a config callback row keeps the test/edit/delete actions and an em-dash access", () => { + const { getByText } = render( + , + ); + const row = getByText("datadog").closest("tr") as HTMLElement; + const scoped = within(row); + expect(scoped.getByTestId("test-callback")).toBeInTheDocument(); + expect(scoped.getByTestId("edit-callback")).toBeInTheDocument(); + expect(scoped.getByTestId("delete-callback")).toBeInTheDocument(); + expect(scoped.queryByTestId("edit-access")).not.toBeInTheDocument(); + // access cell renders an em-dash for non-destination rows + expect(scoped.getByText("—")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 70ec6599ca2..82555ce8929 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -1,6 +1,6 @@ import { Button } from "@tremor/react"; import type { TableProps } from "antd"; -import { Table } from "antd"; +import { Table, Tag } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; @@ -19,9 +19,44 @@ type LoggingCallbacksProps = { onTest?: (callback: AlertingObject) => void | Promise; onEdit?: (callback: AlertingObject) => void; onDelete?: (callback: AlertingObject) => void; + onEditAccess?: (callback: AlertingObject) => void; onAdd?: () => void; }; +const isDestination = (record: AlertingObject): boolean => record.credentialName != null; + +const SCOPE_BADGES_LIMIT = 4; + +// Renders the union of identities that route to this destination, with each team/org +// labeled by its alias. Global supersedes everything. Pulls from record.resolvedScope +// (computed at the page level from BOTH directions: destination-side access AND +// identity-side metadata.logging_exporters). +const ScopeCell: React.FC<{ record: AlertingObject }> = ({ record }) => { + const scope = record.resolvedScope; + if (!scope || (!scope.global && scope.teams.length === 0 && scope.orgs.length === 0)) { + return —; + } + if (scope.global) { + return Global; + } + const items = [ + ...scope.teams.map((label) => ({ kind: "team" as const, label })), + ...scope.orgs.map((label) => ({ kind: "org" as const, label })), + ]; + const shown = items.slice(0, SCOPE_BADGES_LIMIT); + const remainder = items.length - shown.length; + return ( +
+ {shown.map((item, i) => ( + + {item.kind}: {item.label} + + ))} + {remainder > 0 && +{remainder} more} +
+ ); +}; + type CallbackRow = AlertingObject & { id?: string; mode?: "success" | "failure" | "info" | string; @@ -39,6 +74,7 @@ export const LoggingCallbacksTable: React.FC = ({ onTest = () => {}, onEdit = () => {}, onDelete = () => {}, + onEditAccess = () => {}, onAdd = () => {}, }) => { const columns: TableProps["columns"] = [ @@ -49,13 +85,21 @@ export const LoggingCallbacksTable: React.FC = ({ render: (_: string, record: CallbackRow) => { const id = record.name; const displayName = availableCallbacks[id]?.ui_callback_name || id; - return
{displayName}
; + return ( +
+
{displayName}
+ {record.destinationLabel &&
{record.destinationLabel}
} +
+ ); }, }, { title: Mode, key: "mode", render: (_: unknown, record: CallbackRow) => { + // Destination rows fan out on every span, so the success/failure split + // does not apply -- only config callbacks carry a mode. + if (isDestination(record)) return —; // Backend sends `type` (success | failure); legacy in-memory rows // from add-callback flow set `mode`. Read both so newly-added rows // and server-fetched rows both render correctly. @@ -73,20 +117,58 @@ export const LoggingCallbacksTable: React.FC = ({ ); }, - width: 240, + width: 200, + }, + { + title: Scope, + key: "access", + render: (_: unknown, record: CallbackRow) => + isDestination(record) ? : —, + width: 280, }, { title: Actions, key: "actions", align: "right", - render: (_: unknown, record: CallbackRow) => ( -
- onTest(record)} /> - onEdit(record)} /> - onDelete(record)} /> -
- ), - width: 240, + render: (_: unknown, record: CallbackRow) => + isDestination(record) ? ( +
+ onEditAccess(record)} + /> + onDelete(record)} + /> +
+ ) : ( +
+ onTest(record)} + /> + onEdit(record)} + /> + onDelete(record)} + /> +
+ ), + width: 200, }, ]; return ( diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts index 5d265f95484..3bf7b09e47c 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/types.ts @@ -8,6 +8,29 @@ export interface AlertingObject { // every row to render as "Success". type?: "success" | "failure" | "success_and_failure"; variables: AlertingVariables; + // Present only on rows backed by a logging credential (an OTEL trace + // destination). Config-callback rows leave these unset, which is how the table + // tells the two apart. + credentialName?: string; + destinationLabel?: string; + access?: CredentialAccess; + // The union of identities that route to this destination, resolved at render + // time from both directions (destination-side credential_info.access AND + // identity-side metadata.logging_exporters). Display labels only -- ids are + // not surfaced here. global=true bypasses the lists. + resolvedScope?: ResolvedScope; +} + +export interface CredentialAccess { + global?: boolean; + teams?: string[]; + orgs?: string[]; +} + +export interface ResolvedScope { + global: boolean; + teams: string[]; + orgs: string[]; } export interface AlertingVariables { diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index 3c6b3829fef..ab09968c4dd 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -3,7 +3,7 @@ interface CallbackConfig { displayName: string; logo: string; supports_key_team_logging: boolean; - dynamic_params: Record; + dynamic_params: Record; description: string; } @@ -14,11 +14,10 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ id: "arize", displayName: "Arize", logo: `${asset_logos_folder}arize.png`, - supports_key_team_logging: true, - dynamic_params: { - arize_api_key: "password", - arize_space_id: "password", - }, + // OTEL v2 destination: assigned per identity via the "Logging Exporters" field + // (metadata.logging_exporters), not configured as a per-team callback here. + supports_key_team_logging: false, + dynamic_params: {}, description: "Arize Logging Integration", }, { @@ -96,14 +95,22 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ id: "langfuse_otel", displayName: "Langfuse OTEL", logo: `${asset_logos_folder}langfuse.png`, - supports_key_team_logging: true, - dynamic_params: { - langfuse_public_key: "text", - langfuse_secret_key: "password", - langfuse_host: "text", - }, + // OTEL v2 destination: assigned per identity via the "Logging Exporters" field + // (metadata.logging_exporters), not configured as a per-team callback here. + supports_key_team_logging: false, + dynamic_params: {}, description: "Langfuse v3 OTEL Logging Integration", }, + { + id: "weave_otel", + displayName: "Weave OTEL", + logo: `${asset_logos_folder}weave.png`, + // OTEL v2 destination: assigned per identity via the "Logging Exporters" field + // (metadata.logging_exporters), not configured as a per-team callback here. + supports_key_team_logging: false, + dynamic_params: {}, + description: "Weave (W&B) OTEL Logging Integration", + }, { id: "langsmith", displayName: "LangSmith", diff --git a/ui/litellm-dashboard/src/components/logging_credentials/AccessControlFields.tsx b/ui/litellm-dashboard/src/components/logging_credentials/AccessControlFields.tsx new file mode 100644 index 00000000000..d30d27972c2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/logging_credentials/AccessControlFields.tsx @@ -0,0 +1,73 @@ +import { Form, Select, Switch } from "antd"; +import React from "react"; + +import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { CredentialAccess } from "../Settings/LoggingAndAlerts/LoggingCallbacks/types"; + +interface AccessControlFieldsProps { + // value/onChange are optional so the component can be driven either directly + // (the Add modal) or injected by an antd Form.Item (the Edit modal). + value?: CredentialAccess; + onChange?: (next: CredentialAccess) => void; +} + +// Admin-owned access for a logging destination: global (every request) or a set of +// teams/orgs. Per-key targeting is intentionally absent here -- it lives on the key's +// own page, since a key's token rotates on regenerate while team/org ids are stable. +const AccessControlFields: React.FC = ({ value = {}, onChange = () => {} }) => { + const { data: teams } = useTeams(); + const { data: orgs } = useOrganizations(); + const isGlobal = value.global === true; + + const teamOptions = (teams ?? []).map((t) => ({ value: t.team_id, label: t.team_alias || t.team_id })); + const orgOptions = (orgs ?? []).map((o) => ({ + value: o.organization_id, + label: o.organization_alias || o.organization_id, + })); + + return ( + <> + + onChange({ ...value, global })} /> + + + onChange({ ...value, orgs: orgIds })} + options={orgOptions} + optionFilterProp="label" + style={{ width: "100%" }} + /> + + + ); +}; + +export default AccessControlFields; diff --git a/ui/litellm-dashboard/src/components/logging_credentials/EditLoggingCredentialModal.tsx b/ui/litellm-dashboard/src/components/logging_credentials/EditLoggingCredentialModal.tsx new file mode 100644 index 00000000000..79680a5740a --- /dev/null +++ b/ui/litellm-dashboard/src/components/logging_credentials/EditLoggingCredentialModal.tsx @@ -0,0 +1,74 @@ +import { Form, Modal } from "antd"; +import React from "react"; + +import { CredentialAccess } from "../Settings/LoggingAndAlerts/LoggingCallbacks/types"; +import NotificationsManager from "../molecules/notifications_manager"; +import { credentialUpdateCall } from "../networking"; +import AccessControlFields from "./AccessControlFields"; + +interface EditLoggingCredentialModalProps { + accessToken: string; + credentialName: string | null; + access?: CredentialAccess; + open: boolean; + onClose: () => void; + onSaved: () => void; +} + +interface AccessForm { + access?: CredentialAccess; +} + +const EditLoggingCredentialModal: React.FC = ({ + accessToken, + credentialName, + access, + open, + onClose, + onSaved, +}) => { + // destroyOnClose remounts the Form each open, so initialValues re-seeds from the + // current destination -- no effect syncing prop into state. + const [form] = Form.useForm(); + + const handleSave = async () => { + if (!credentialName) return; + const current = form.getFieldsValue().access ?? {}; + // Always send the full access object: credential_info merges server-side, so a + // sparse patch could never clear a bucket. A global grant supersedes team/org. + const next: CredentialAccess = current.global + ? { global: true, teams: [], orgs: [] } + : { global: false, teams: current.teams ?? [], orgs: current.orgs ?? [] }; + try { + await credentialUpdateCall(accessToken, credentialName, { + credential_name: credentialName, + credential_values: {}, + credential_info: { access: next }, + }); + NotificationsManager.success("Access updated"); + onSaved(); + onClose(); + } catch (error) { + NotificationsManager.fromBackend(error instanceof Error ? error.message : String(error)); + } + }; + + return ( + + form={form} layout="vertical" preserve={false} initialValues={{ access: access ?? {} }}> + + + + + + ); +}; + +export default EditLoggingCredentialModal; diff --git a/ui/litellm-dashboard/src/components/logging_credentials/LoggingExportersSelect.test.tsx b/ui/litellm-dashboard/src/components/logging_credentials/LoggingExportersSelect.test.tsx new file mode 100644 index 00000000000..a97b59c7b7a --- /dev/null +++ b/ui/litellm-dashboard/src/components/logging_credentials/LoggingExportersSelect.test.tsx @@ -0,0 +1,158 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import LoggingExportersSelect from "./LoggingExportersSelect"; + +const mockUseCredentials = vi.fn(); +const mockUseAuthorized = vi.fn(); +const mockUseTeams = vi.fn(); +const mockUseOrganizations = vi.fn(); + +vi.mock("@/app/(dashboard)/hooks/credentials/useCredentials", () => ({ + useCredentials: () => mockUseCredentials(), +})); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useTeams: () => mockUseTeams(), +})); +vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ + useOrganizations: () => mockUseOrganizations(), +})); + +vi.mock("antd", async () => { + const React = await import("react"); + function Select(props: any) { + const { value, onChange, options, notFoundContent } = props; + return React.createElement( + "div", + { "data-testid": "logging-exporters-select" }, + React.createElement( + "ul", + null, + (options ?? []).map((opt: any) => + React.createElement("li", { key: opt.value, "data-testid": "option" }, opt.label), + ), + ), + options && options.length === 0 ? React.createElement("div", { "data-testid": "empty" }, notFoundContent) : null, + React.createElement( + "button", + { "data-testid": "pick-first", onClick: () => onChange?.(options?.[0] ? [options[0].value] : []) }, + "pick first", + ), + React.createElement("div", { "data-testid": "value" }, JSON.stringify(value ?? [])), + ); + } + return { Select }; +}); + +beforeEach(() => { + // Default: a proxy admin (formatted role "Admin"), no team/org membership needed. + mockUseAuthorized.mockReturnValue({ userRole: "Admin" }); + mockUseTeams.mockReturnValue({ data: [] }); + mockUseOrganizations.mockReturnValue({ data: [] }); +}); + +describe("LoggingExportersSelect", () => { + it("only surfaces credentials whose credential_type is 'logging'", () => { + mockUseCredentials.mockReturnValue({ + data: { + credentials: [ + { + credential_name: "poc-langfuse", + credential_info: { credential_type: "logging", host: "https://cloud.langfuse.com" }, + }, + { + credential_name: "poc-arize", + credential_info: { credential_type: "logging" }, + }, + { + credential_name: "openai-prod", + credential_info: { custom_llm_provider: "openai" }, + }, + ], + }, + }); + + render( {}} />); + + const options = screen.getAllByTestId("option").map((el) => el.textContent); + expect(options).toEqual(["poc-langfuse (https://cloud.langfuse.com)", "poc-arize"]); + }); + + it("renders empty-state copy when no logging destinations exist", () => { + mockUseCredentials.mockReturnValue({ + data: { + credentials: [ + { + credential_name: "openai-prod", + credential_info: { custom_llm_provider: "openai" }, + }, + ], + }, + }); + + render( {}} />); + + expect(screen.queryAllByTestId("option")).toHaveLength(0); + expect(screen.getByTestId("empty").textContent).toMatch(/proxy admin/i); + }); + + it("scopes a non-admin caller to destinations granted to their team/org (plus global/auto_enable)", () => { + // An internal_user who is a member of team-a. They must see only what they could + // actually assign: the team-a destination, the global one, and the auto_enable + // default -- never the team-b destination or the foreign-org one. This mirrors the + // backend assignment gate; the backend stays the authoritative check. + mockUseAuthorized.mockReturnValue({ userRole: "Internal User" }); + mockUseTeams.mockReturnValue({ data: [{ team_id: "team-a" }] }); + mockUseOrganizations.mockReturnValue({ data: [{ organization_id: "org-a" }] }); + mockUseCredentials.mockReturnValue({ + data: { + credentials: [ + { + credential_name: "mine-team", + credential_info: { credential_type: "logging", access: { teams: ["team-a"] } }, + }, + { + credential_name: "foreign-team", + credential_info: { credential_type: "logging", access: { teams: ["team-b"] } }, + }, + { credential_name: "mine-org", credential_info: { credential_type: "logging", access: { orgs: ["org-a"] } } }, + { + credential_name: "foreign-org", + credential_info: { credential_type: "logging", access: { orgs: ["org-z"] } }, + }, + { credential_name: "everyone", credential_info: { credential_type: "logging", access: { global: true } } }, + { credential_name: "always-on", credential_info: { credential_type: "logging", auto_enable: true } }, + ], + }, + }); + + render( {}} />); + + const options = screen.getAllByTestId("option").map((el) => el.textContent); + expect(options).toEqual(["mine-team", "mine-org", "everyone", "always-on"]); + }); + + it("shows every logging destination to a proxy admin regardless of access scope", () => { + mockUseAuthorized.mockReturnValue({ userRole: "Admin" }); + mockUseTeams.mockReturnValue({ data: [] }); + mockUseCredentials.mockReturnValue({ + data: { + credentials: [ + { + credential_name: "team-b-only", + credential_info: { credential_type: "logging", access: { teams: ["team-b"] } }, + }, + { credential_name: "no-access", credential_info: { credential_type: "logging" } }, + ], + }, + }); + + render( {}} />); + + const options = screen.getAllByTestId("option").map((el) => el.textContent); + expect(options).toEqual(["team-b-only", "no-access"]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/logging_credentials/LoggingExportersSelect.tsx b/ui/litellm-dashboard/src/components/logging_credentials/LoggingExportersSelect.tsx new file mode 100644 index 00000000000..6738ec7ae8a --- /dev/null +++ b/ui/litellm-dashboard/src/components/logging_credentials/LoggingExportersSelect.tsx @@ -0,0 +1,72 @@ +import { Select } from "antd"; +import React from "react"; + +import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials"; +import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { isAdminRole } from "@/utils/roles"; + +interface LoggingExportersSelectProps { + value?: string[]; + onChange?: (value: string[]) => void; +} + +/** + * Multi-select of admin-owned logging destinations (credential_type=logging) that an + * identity (key / team / org) exports its traces to. The selected names are stored in + * metadata.logging_exporters; the proxy unions them across the identity chain and fans + * out. + * + * Options are scoped to what the caller can actually assign: a proxy admin sees every + * destination; everyone else sees only the ones visible to a team or org they belong to + * (plus global / auto_enable destinations). This mirrors the backend assignment gate so + * a team admin is not offered another tenant's destination only to have the save + * rejected, and it avoids surfacing other tenants' destination names. The backend stays + * the authoritative check -- this filter is UX, not a security boundary. + */ +const LoggingExportersSelect: React.FC = ({ value, onChange }) => { + const { data } = useCredentials(); + const { userRole } = useAuthorized(); + const { data: teams } = useTeams(); + const { data: orgs } = useOrganizations(); + + const seesEveryDestination = isAdminRole(userRole ?? ""); + const myTeamIds = new Set((teams ?? []).map((t) => t.team_id)); + const myOrgIds = new Set((orgs ?? []).map((o) => o.organization_id)); + + const assignable = (info: { + auto_enable?: boolean; + access?: { global?: boolean; teams?: string[]; orgs?: string[] }; + }) => { + if (info.auto_enable === true || info.access?.global === true) return true; + if ((info.access?.teams ?? []).some((id) => myTeamIds.has(id))) return true; + return (info.access?.orgs ?? []).some((id) => myOrgIds.has(id)); + }; + + const options = (data?.credentials ?? []) + .filter((credential) => credential.credential_info?.credential_type === "logging") + .filter((credential) => seesEveryDestination || assignable(credential.credential_info)) + .map((credential) => ({ + value: credential.credential_name, + label: credential.credential_info?.host + ? `${credential.credential_name} (${credential.credential_info.host})` + : credential.credential_name, + })); + + return ( + + + {addingDestinationFields.map((f) => ( + {f.label}} + name={f.name} + rules={ + f.optional ? undefined : [{ required: true, message: `Please enter the ${f.label.toLowerCase()}` }] + } + > + {f.type === "password" ? ( + + ) : ( + + )} + + ))} + + + + +
+ ) : ( + + )}
= ({ accessToken, userRole, userID, setShowAddCallbacksModal(false); setSelectedCallback(null); setSelectedCallbackParams([]); + setAddAccess({}); + setAddAutoEnable(false); addForm.resetFields(); }} disabled={isAddingCallback} @@ -749,7 +920,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, Cancel - {isAddingCallback ? "Adding..." : "Add Callback"} + {isAddingCallback ? "Adding..." : "Add"}
diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 46eac1c5772..244fb190f90 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -31,6 +31,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ useTeam: vi.fn(), + useTeams: vi.fn().mockReturnValue({ data: [] }), })); vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 602490a0c98..e5902800899 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -41,6 +41,7 @@ import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSel import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; import GuardrailSettingsView from "../GuardrailSettingsView"; import LoggingSettingsView from "../logging_settings_view"; +import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; import { ModelSelect } from "../ModelSelect/ModelSelect"; @@ -51,6 +52,7 @@ import NumericalInput from "../shared/numerical_input"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; import SearchToolSelector from "../SearchTools/SearchToolSelector"; import EditLoggingSettings from "./EditLoggingSettings"; +import LoggingExportersSelect from "../logging_credentials/LoggingExportersSelect"; import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion"; import MemberModal from "./EditMembership"; import MemberPermissions from "./member_permissions"; @@ -233,6 +235,25 @@ const TeamInfoView: React.FC = ({ }, [selectedModelsInForm, teamData, userModels]); const canEditTeam = is_team_admin || is_proxy_admin || is_org_admin || isOrgAdminForTeam; + + // Destinations whose credential_info.access targets this team (or its org, or + // global). Rendered alongside the team's own metadata.logging_exporters so the + // Logging Exporters card reflects BOTH routing directions, matching the + // resolver's union at request time. + const { data: scopedCredentialsData } = useCredentials(); + const scopedExportersForTeam = useMemo(() => { + const orgId = teamData?.team_info?.organization_id ?? null; + return (scopedCredentialsData?.credentials ?? []) + .filter((c) => c.credential_info?.credential_type === "logging") + .filter((c) => { + const access = c.credential_info?.access; + if (!access) return false; + if (access.global === true) return true; + if (Array.isArray(access.teams) && access.teams.includes(teamId)) return true; + return Array.isArray(access.orgs) && orgId != null && access.orgs.includes(orgId); + }) + .map((c) => c.credential_name); + }, [scopedCredentialsData?.credentials, teamId, teamData?.team_info?.organization_id]); const visibleTabs = useMemo(() => getTeamInfoVisibleTabs(canEditTeam), [canEditTeam]); const defaultTabKey = useMemo(() => getTeamInfoDefaultTab(editTeam, canEditTeam), [editTeam, canEditTeam]); @@ -530,6 +551,7 @@ const TeamInfoView: React.FC = ({ guardrails: (values.guardrails || []).filter((n: string) => !globalGuardrailNames.has(n)), opted_out_global_guardrails: optedOutGlobalGuardrails, ...(values.logging_settings?.length > 0 ? { logging: values.logging_settings } : {}), + ...(values.logging_exporters !== undefined ? { logging_exporters: values.logging_exporters } : {}), disable_global_guardrails: killSwitchOnAtSave, soft_budget_alerting_emails: typeof values.soft_budget_alerting_emails === "string" @@ -541,7 +563,11 @@ const TeamInfoView: React.FC = ({ ...(secretManagerSettings !== undefined ? { secret_manager_settings: secretManagerSettings } : {}), }, ...(values.policies?.length > 0 ? { policies: values.policies } : {}), - ...(values.organization_id !== info.organization_id ? { organization_id: values.organization_id ?? null } : {}), + // organization_id always sent so the backend route gate can identify + // org-admin callers (the gate matches on body.organization_id; omitting + // it falls through to default-deny for a non-PROXY_ADMIN caller even + // if they admin the team's current org). + organization_id: values.organization_id !== undefined ? values.organization_id ?? null : info.organization_id, }; updateData.max_budget = mapEmptyStringToNull(updateData.max_budget); @@ -862,6 +888,10 @@ const TeamInfoView: React.FC = ({ @@ -974,6 +1004,7 @@ const TeamInfoView: React.FC = ({ ) : "", logging_settings: info.metadata?.logging || [], + logging_exporters: info.metadata?.logging_exporters || [], secret_manager_settings: info.metadata?.secret_manager_settings ? JSON.stringify(info.metadata.secret_manager_settings, null, 2) : "", @@ -1425,6 +1456,14 @@ const TeamInfoView: React.FC = ({ /> + + + + = ({ ({ }), })); +// KeyInfoView's Logging Exporters select pulls credentials + orgs via react-query; +// mock both so this QueryClientProvider-free unit test of handleKeyUpdate runs. +vi.mock("@/app/(dashboard)/hooks/credentials/useCredentials", () => ({ + useCredentials: vi.fn().mockReturnValue({ data: { credentials: [] }, refetch: vi.fn() }), +})); + +vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ + useOrganizations: vi.fn().mockReturnValue({ data: [], refetch: vi.fn() }), +})); + // KeyEditView mock: triggers onSubmit with our injected form values vi.mock("./key_edit_view", async () => { const React = await import("react"); diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index e991db8069e..8f7a2f7393a 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -27,6 +27,7 @@ import { fetchTeamModels } from "../organisms/create_key_button"; import NumericalInput from "../shared/numerical_input"; import { Tag } from "../tag_management/types"; import EditLoggingSettings from "../team/EditLoggingSettings"; +import LoggingExportersSelect from "../logging_credentials/LoggingExportersSelect"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; interface KeyEditViewProps { @@ -190,6 +191,7 @@ export function KeyEditView({ accessGroups: keyData.object_permission?.agent_access_groups || [], }, logging_settings: extractLoggingSettings(keyData.metadata), + logging_exporters: keyData.metadata?.logging_exporters || [], disabled_callbacks: Array.isArray(keyData.metadata?.litellm_disabled_callbacks) ? mapInternalToDisplayNames(keyData.metadata.litellm_disabled_callbacks) : [], @@ -219,6 +221,7 @@ export function KeyEditView({ }, mcp_tool_permissions: keyData.object_permission?.mcp_tool_permissions || {}, logging_settings: extractLoggingSettings(keyData.metadata), + logging_exporters: keyData.metadata?.logging_exporters || [], disabled_callbacks: Array.isArray(keyData.metadata?.litellm_disabled_callbacks) ? mapInternalToDisplayNames(keyData.metadata.litellm_disabled_callbacks) : [], @@ -731,6 +734,14 @@ export function KeyEditView({ )} + + + + (keyData); + + // Destinations whose credential_info.access targets THIS key (via its team_id, + // its team's organization_id, or global). Rendered alongside the key's own + // metadata.logging_exporters so the Logging Exporters section reflects BOTH + // routing directions, matching the resolver's union at request time. + const scopedExportersForKey = useMemo(() => { + const keyTeamId = (currentKeyData as { team_id?: string | null } | undefined)?.team_id ?? null; + const team = (teamsData ?? []).find((t) => t.team_id === keyTeamId); + const teamOrgId = (team as { organization_id?: string | null } | undefined)?.organization_id ?? null; + return (keyCredentialsData?.credentials ?? []) + .filter((c) => c.credential_info?.credential_type === "logging") + .filter((c) => { + const access = c.credential_info?.access; + if (!access) return false; + if (access.global === true) return true; + if (Array.isArray(access.teams) && keyTeamId && access.teams.includes(keyTeamId)) return true; + return Array.isArray(access.orgs) && teamOrgId != null && access.orgs.includes(teamOrgId); + }) + .map((c) => c.credential_name); + }, [keyCredentialsData?.credentials, currentKeyData, teamsData, keyOrganizationsData]); const [lastRegeneratedAt, setLastRegeneratedAt] = useState(null); const [isRecentlyRegenerated, setIsRecentlyRegenerated] = useState(false); const [policyGuardrails, setPolicyGuardrails] = useState>({}); @@ -254,6 +278,7 @@ export default function KeyInfoView({ ...(Array.isArray(formValues.logging_settings) && formValues.logging_settings.length > 0 ? { logging: formValues.logging_settings } : {}), + ...(formValues.logging_exporters !== undefined ? { logging_exporters: formValues.logging_exporters } : {}), ...(formValues.disabled_callbacks?.length > 0 ? { litellm_disabled_callbacks: mapDisplayToInternalNames(formValues.disabled_callbacks), @@ -275,6 +300,7 @@ export default function KeyInfoView({ ...(Array.isArray(formValues.logging_settings) && formValues.logging_settings.length > 0 ? { logging: formValues.logging_settings } : {}), + ...(formValues.logging_exporters !== undefined ? { logging_exporters: formValues.logging_exporters } : {}), ...(formValues.disabled_callbacks?.length > 0 ? { litellm_disabled_callbacks: mapDisplayToInternalNames(formValues.disabled_callbacks), @@ -617,6 +643,12 @@ export default function KeyInfoView({