diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 17011bb8db7..3038bdb90b2 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -223,6 +223,15 @@ lives in [`plumbing/`](./plumbing): readers/exporters receive them alongside the server metrics, and one is built and registered as the global only when none is set (mirroring how V2 owns trace export). +- [`events.py`](./plumbing/events.py) — GenAI client events. Gated on + `enable_events` (`LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS`), a failed LLM call + records the semconv `gen_ai.client.operation.exception` log event at severity + WARN, carrying `exception.type` / `exception.message` / `exception.stacktrace` + and correlated to the failed span through the trace and span ids. The + `LoggerProvider` is resolved like the meter provider, except that an explicit + `NoOpLoggerProvider` global is an operator opt-out that builds no recorder at + all. The deprecated `error.*` span attributes and the `exception` span event + are still stamped by the emitter for backwards compatibility. ### Adapter diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 1edad24389f..8c4eba522db 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -17,6 +17,7 @@ from litellm.integrations.otel.model.payloads import ( ServiceSpanData, SpanError, ) +from litellm.integrations.otel.plumbing.events import GenAIEventRecorder from litellm.integrations.otel.plumbing.providers import to_otel_span_kind from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError from litellm.integrations.otel.model.spans import ( @@ -74,9 +75,11 @@ class SpanEmitter: tracer: Tracer, config: OpenTelemetryV2Config, mappers: Sequence[AttributeMapper] | None = None, + event_recorder: GenAIEventRecorder | None = None, ) -> None: self._tracer = tracer self._config = config + self._event_recorder = event_recorder # The mapper chain is the sole source of span attributes. When not # passed in, resolve it from the config so there's one source of truth. self._mappers: list[AttributeMapper] = ( @@ -208,6 +211,14 @@ class SpanEmitter: ExceptionEvent.NAME, {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message}, ) + if self._event_recorder is not None and role is SpanRole.LLM_CALL: + self._event_recorder.record_operation_exception( + span_context=span.get_span_context(), + error_type=error_type, + message=message, + stack_trace=error.stack_trace, + timestamp_ns=end_time_ns, + ) # On success leave the status UNSET (the semconv default) rather than # forcing OK — that matches the FastAPI server span and avoids implying a # span-level health signal litellm doesn't actually evaluate. Only a diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 44484559948..fe5b0ae776e 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, cast from opentelemetry.context import attach, get_current +from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import Span, Tracer, get_current_span, use_span @@ -37,14 +38,17 @@ from litellm.integrations.otel.model.payloads import ( SpanError, is_mcp_tool_call, ) +from litellm.integrations.otel.plumbing.events import GenAIEventRecorder from litellm.integrations.otel.plumbing.metrics import ( GenAIMetricRecorder, create_genai_metrics, ) from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, + get_event_logger, get_meter, get_tracer, + resolve_logger_provider, resolve_meter_provider, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache @@ -101,7 +105,7 @@ class OpenTelemetryV2(CustomLogger): config: OpenTelemetryV2Config | None = None, callback_name: str | None = None, tracer_provider: TracerProvider | None = None, - logger_provider: Any | None = None, # reserved for OTel logs + logger_provider: LoggerProvider | None = None, meter_provider: Any | None = None, **kwargs: Any, ) -> None: @@ -114,7 +118,12 @@ class OpenTelemetryV2(CustomLogger): self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) self._metric_filter_error_logged = False - self._emitter = SpanEmitter(self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)) + self._emitter = SpanEmitter( + self.tracer, + self.config, + mappers=resolve_mappers(self.config.mapper_names), + event_recorder=self._init_events(logger_provider), + ) self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME) self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict() self._init_otel_logger_on_litellm_proxy() @@ -133,6 +142,22 @@ class OpenTelemetryV2(CustomLogger): meter = get_meter(provider, LITELLM_TRACER_NAME) return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name) + def _init_events(self, logger_provider: LoggerProvider | None) -> "GenAIEventRecorder | None": + """Create the GenAI event recorder when events are enabled, else ``None``. + + ``logger_provider`` is an explicit override (tests inject one); otherwise the + provider is resolved from the OTel global so an operator-configured logs + pipeline receives the events, building and registering one only when no + global provider is set. A ``None`` resolution means the operator opted out + of the logs signal, so no recorder is built. + """ + if not self.config.enable_events: + return None + provider = resolve_logger_provider(self.config, logger_provider) + if provider is None: + return None + return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME)) + # ====================================================================== # # Proxy global registration # ====================================================================== # diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 1fcfd98bed5..4a09e08cbc3 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -179,6 +179,19 @@ class ExceptionEvent: NAME: Final = "exception" TYPE: Final = "exception.type" MESSAGE: Final = "exception.message" + STACKTRACE: Final = "exception.stacktrace" + + +class GenAIEvent: + """GenAI semconv event names, from the GenAI registry's *events* section. + + ``gen_ai.client.operation.exception`` is defined as a log-based event + (severity WARN) carrying the ``exception.*`` trio, correlated to the failed + span via the trace/span ids — the semconv-compliant home for GenAI failure + details, unlike the deprecated ``error.message`` span attribute. + """ + + OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception" class Server: diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py new file mode 100644 index 00000000000..f674526d04f --- /dev/null +++ b/litellm/integrations/otel/plumbing/events.py @@ -0,0 +1,52 @@ +"""GenAI client events: the ``gen_ai.client.operation.exception`` log event. + +The GenAI semantic conventions define exception recording for client +operations as a log-based event (severity WARN) carrying the ``exception.*`` +attribute trio, correlated to the failed span through the trace/span ids — +not as a span attribute or span event. This module owns building and +emitting that event; the exporter pipeline it rides is built in +:mod:`litellm.integrations.otel.plumbing.providers`. +""" + +from dataclasses import dataclass + +from opentelemetry._events import Event, EventLogger +from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.trace import SpanContext + +from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + +@dataclass(frozen=True, slots=True) +class GenAIEventRecorder: + event_logger: EventLogger + + def record_operation_exception( + self, + span_context: SpanContext, + error_type: str, + message: str, + stack_trace: str | None, + timestamp_ns: int | None, + ) -> None: + # ``exception.type`` and ``exception.message`` are the semconv-required + # pair and always ride the event; only the recommended stacktrace is + # conditional on the payload carrying one. + stacktrace = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else () + self.event_logger.emit( + Event( + name=GenAIEvent.OPERATION_EXCEPTION, + timestamp=timestamp_ns, + trace_id=span_context.trace_id, + span_id=span_context.span_id, + trace_flags=span_context.trace_flags, + severity_number=SeverityNumber.WARN, + attributes=dict( + ( + (ExceptionEvent.TYPE, error_type), + (ExceptionEvent.MESSAGE, message), + *stacktrace, + ) + ), + ) + ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ac971c6daa8..ced65aa1ec3 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -2,9 +2,20 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable -from opentelemetry import baggage, metrics +from opentelemetry import _logs, baggage, metrics +from opentelemetry._events import EventLogger +from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context from opentelemetry.metrics import MeterProvider, NoOpMeterProvider +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider +from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, + ConsoleLogExporter, + InMemoryLogExporter, + LogExporter, + SimpleLogRecordProcessor, +) from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider @@ -224,6 +235,112 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) +def _otlp_logs_endpoint(endpoint: str | None) -> str | None: + """Point an OTLP/HTTP base endpoint at the ``/v1/logs`` signal path. + + The OTLP/HTTP exporter only appends ``/v1/logs`` when it reads + ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used + verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint`` + for the logs signal (rewriting a sibling signal path when present). + """ + if not endpoint: + return endpoint + endpoint = endpoint.rstrip("/") + if endpoint.endswith("/v1/logs"): + return endpoint + for other_signal in ("/v1/traces", "/v1/metrics"): + if endpoint.endswith(other_signal): + return endpoint[: -len(other_signal)] + "/v1/logs" + return endpoint + "/v1/logs" + + +def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter: + """Build a log exporter mirroring the exporter selection of the other signals. + + ``console`` (and any unrecognized kind) exports to the console; ``otlp_http`` + and ``otlp_grpc`` export over OTLP with the configured endpoint/headers; + ``in_memory`` buffers for tests. Like GenAI metrics, events ride the + single-destination shorthand fields, not the multi-exporter ``exporters`` list. + """ + kind = (config.exporter or "console").lower() + if kind in ("in_memory", "inmemory", "memory"): + return InMemoryLogExporter() + if kind in ("otlp_http", "http", "http/protobuf", "http/json"): + from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter as HTTPLogExporter, + ) + + return HTTPLogExporter( + endpoint=_otlp_logs_endpoint(config.endpoint), + headers=parse_headers(config.headers), + ) + if kind in ("otlp_grpc", "grpc"): + try: + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter as GRPCLogExporter, + ) + except ImportError as exc: + raise ImportError( + "OpenTelemetry OTLP gRPC log exporter is not available. Install " + "`opentelemetry-exporter-otlp` and `grpcio` (or `litellm[grpc]`)." + ) from exc + + return GRPCLogExporter(endpoint=config.endpoint, headers=parse_headers(config.headers)) + return ConsoleLogExporter() + + +def build_logger_provider( + config: OpenTelemetryV2Config, + log_exporter: LogExporter | None = None, +) -> SDKLoggerProvider: + """Build the :class:`LoggerProvider` GenAI events export through. + + ``log_exporter`` is an explicit override (tests inject an + ``InMemoryLogExporter``); otherwise the exporter is selected from the config's + exporter kind via :func:`build_log_exporter`. Console and in-memory exporters + get a Simple processor (synchronous export, which tests rely on), everything + else a Batch processor — the same split as span processing. + """ + exporter = log_exporter if log_exporter is not None else build_log_exporter(config) + provider = SDKLoggerProvider(resource=build_resource(config)) + use_simple = isinstance(exporter, (ConsoleLogExporter, InMemoryLogExporter)) + provider.add_log_record_processor( + SimpleLogRecordProcessor(exporter) if use_simple else BatchLogRecordProcessor(exporter) + ) + return provider + + +def resolve_logger_provider( + config: OpenTelemetryV2Config, + logger_provider: SDKLoggerProvider | None = None, +) -> SDKLoggerProvider | None: + """Resolve the :class:`LoggerProvider` GenAI events record through, or ``None`` + when the operator has opted out of the logs signal. + + Same resolution order as :func:`resolve_meter_provider`: an injected provider + wins (DI/tests); an operator-configured SDK global is reused so events ride + their pipeline; an explicit ``NoOpLoggerProvider`` global is an opt-out and + yields ``None``, so no event is ever built. Only the default placeholder + global makes V2 build a provider from the config and publish it as the global. + """ + if logger_provider is not None: + return logger_provider + + existing: LoggerProvider = _logs.get_logger_provider() + if isinstance(existing, SDKLoggerProvider): + return existing + if isinstance(existing, NoOpLoggerProvider): + return None + + provider = build_logger_provider(config) + _logs.set_logger_provider(provider) + return provider + + +def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger: + return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version) + + def build_meter_provider( config: OpenTelemetryV2Config, metric_reader: "MetricReader | None" = None, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 298047ec18b..8f993aa385e 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -485,6 +485,74 @@ def test_build_span_exporter_variants(): OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") ) assert "OTLPSpanExporter" in type(http_exporter).__name__ + + +def test_otlp_logs_endpoint_normalization(): + norm = providers._otlp_logs_endpoint + # A base endpoint gets the signal path appended (the common OTLP env shape). + assert norm("http://collector:4318") == "http://collector:4318/v1/logs" + assert norm("http://collector:4318/") == "http://collector:4318/v1/logs" + # An already-correct path is left intact. + assert norm("http://collector:4318/v1/logs") == "http://collector:4318/v1/logs" + # A sibling signal's path is rewritten to logs, so one OTEL_ENDPOINT works + # for every signal rather than POSTing events at the traces path. + assert norm("http://collector:4318/v1/traces") == "http://collector:4318/v1/logs" + assert norm("http://collector:4318/v1/metrics") == "http://collector:4318/v1/logs" + assert norm(None) is None + + +def test_build_log_exporter_variants(): + from opentelemetry.sdk._logs.export import ConsoleLogExporter, InMemoryLogExporter + + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="console")), + ConsoleLogExporter, + ) + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="in_memory")), + InMemoryLogExporter, + ) + # An unrecognized kind falls back to console rather than dropping events. + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="unknown")), + ConsoleLogExporter, + ) + http_exporter = providers.build_log_exporter( + OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") + ) + assert "OTLPLogExporter" in type(http_exporter).__name__ + + +def test_build_logger_provider_picks_processor_by_exporter_kind(): + """Console and in-memory exporters export synchronously (tests depend on it); + every other destination gets the batch processor.""" + from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, + ConsoleLogExporter, + InMemoryLogExporter, + SimpleLogRecordProcessor, + ) + + cfg = OpenTelemetryV2Config(exporter="in_memory") + + def processor_of(provider): + return provider._multi_log_record_processor._log_record_processors[0] + + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=InMemoryLogExporter())), + SimpleLogRecordProcessor, + ) + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=ConsoleLogExporter())), + SimpleLogRecordProcessor, + ) + http_exporter = providers.build_log_exporter( + OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") + ) + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=http_exporter)), + BatchLogRecordProcessor, + ) grpc_exporter = providers.build_span_exporter( OpenTelemetryV2Config(exporter="otlp_grpc", endpoint="http://h:4317") ) @@ -721,6 +789,177 @@ def test_success_span_records_no_exception_event(): assert all(e.name != ExceptionEvent.NAME for e in span.events) +def _engine_with_event_recorder(): + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + from litellm.integrations.otel.emitter import SpanEmitter + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + provider, span_exporter = providers.in_memory_provider(cfg) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider)) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg, event_recorder=recorder) + return engine, span_exporter, log_exporter + + +def _llm_call_data(error): + return LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model=None, + response_id=None, + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=(), + error=error, + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + + +def test_operation_exception_log_event_emitted_on_failed_llm_call(): + """A failed LLM call records the GenAI semconv ``gen_ai.client.operation.exception`` + event on the logs signal: severity WARN, the full ``exception.*`` trio (including + the stacktrace, which span-side only exists under a vendor key), correlated to + the failed span via trace/span ids. The span-side error surface stays intact.""" + from opentelemetry._logs.severity import SeverityNumber + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.LLM_CALL, + _llm_call_data( + SpanError( + error_type="RateLimitError", + message="rate limited", + code="429", + stack_trace="Traceback (most recent call last) ...", + llm_provider="openai", + ) + ), + ) + (span,) = span_exporter.get_finished_spans() + (log,) = log_exporter.get_finished_logs() + record = log.log_record + + assert record.attributes["event.name"] == GenAIEvent.OPERATION_EXCEPTION + assert record.severity_number == SeverityNumber.WARN + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited" + assert record.attributes[ExceptionEvent.STACKTRACE] == "Traceback (most recent call last) ..." + assert record.trace_id == span.context.trace_id + assert record.span_id == span.context.span_id + + assert [e.name for e in span.events] == [ExceptionEvent.NAME] + assert span.attributes["error.type"] == "RateLimitError" + + +def test_operation_exception_log_event_omits_absent_stacktrace(): + from litellm.integrations.otel.model.semconv import ExceptionEvent + + engine, _, log_exporter = _engine_with_event_recorder() + engine.emit(SpanRole.LLM_CALL, _llm_call_data(SpanError(error_type="APIError", message="boom"))) + (log,) = log_exporter.get_finished_logs() + + assert ExceptionEvent.STACKTRACE not in log.log_record.attributes + assert log.log_record.attributes[ExceptionEvent.MESSAGE] == "boom" + + +def test_operation_exception_log_event_always_carries_required_pair(): + """``exception.type`` and ``exception.message`` are the semconv-required pair: + they ride the event even when the recorder is handed empty strings, so an + event is never emitted with no required field. Only the stacktrace is + conditional.""" + from opentelemetry.sdk._logs.export import InMemoryLogExporter + from opentelemetry.trace import INVALID_SPAN_CONTEXT + + from litellm.integrations.otel.model.semconv import ExceptionEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider)) + + recorder.record_operation_exception( + span_context=INVALID_SPAN_CONTEXT, + error_type="", + message="", + stack_trace="", + timestamp_ns=None, + ) + (log,) = log_exporter.get_finished_logs() + attributes = log.log_record.attributes + assert attributes[ExceptionEvent.TYPE] == "" + assert attributes[ExceptionEvent.MESSAGE] == "" + assert ExceptionEvent.STACKTRACE not in attributes + + +def test_operation_exception_log_event_not_emitted_on_success(): + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) + + assert len(span_exporter.get_finished_spans()) == 1 + assert log_exporter.get_finished_logs() == () + + +def test_operation_exception_log_event_only_for_llm_call_role(): + """The event is scoped to GenAI client operations; a failed guardrail span + keeps its span-side error surface but records no GenAI exception event.""" + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.GUARDRAIL, + GuardrailSpanData("presidio", status="failure", error=SpanError(error_type="X", message="denied")), + ) + (span,) = span_exporter.get_finished_spans() + + assert span.attributes["error.type"] == "X" + assert log_exporter.get_finished_logs() == () + + +def test_resolve_logger_provider_honors_explicit_noop_optout(monkeypatch): + """A ``NoOpLoggerProvider`` global is an explicit operator opt-out from the logs + signal: resolve to ``None`` so no recorder (and so no event) is ever built, + rather than emitting into a provider that drops everything.""" + from opentelemetry import _logs + from opentelemetry._logs import NoOpLoggerProvider + + from litellm.integrations.otel.logger import OpenTelemetryV2 + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + tracer_provider, _ = providers.in_memory_provider(cfg) + monkeypatch.setattr(_logs, "get_logger_provider", lambda: NoOpLoggerProvider()) + + assert providers.resolve_logger_provider(cfg) is None + logger = OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider) + assert logger._emitter._event_recorder is None + + +def test_resolve_logger_provider_reuses_operator_sdk_global(monkeypatch): + """Events ride an operator-configured logs pipeline rather than a second one + built by litellm, so they land wherever the operator's other logs land.""" + from opentelemetry import _logs + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + operator_provider = providers.build_logger_provider(cfg, log_exporter=InMemoryLogExporter()) + monkeypatch.setattr(_logs, "get_logger_provider", lambda: operator_provider) + + assert providers.resolve_logger_provider(cfg) is operator_provider + + +def test_operation_exception_event_keys_are_pinned(): + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + assert GenAIEvent.OPERATION_EXCEPTION == "gen_ai.client.operation.exception" + assert ExceptionEvent.STACKTRACE == "exception.stacktrace" + + # --- service taxonomy: which calls become spans, and of what kind ----------- # diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 0ceb7efbe0b..0e38ddbeab8 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -176,6 +176,61 @@ def test_async_log_failure_event_marks_error_status(): assert span.attributes["error.type"] == "RateLimitError" +def _logger_with_events(enable_events): + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=enable_events) + span_exporter = InMemorySpanExporter() + tracer_provider = providers.build_tracer_provider(cfg, exporter=span_exporter) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + logger = OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider, logger_provider=logger_provider) + return logger, span_exporter, log_exporter + + +def test_enable_events_records_operation_exception_through_failure_callback(): + """With ``enable_events`` on, a real failure callback records the GenAI + ``gen_ai.client.operation.exception`` log event, carrying the traceback from + the standard logging payload and correlated to the LLM-call span.""" + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + logger, span_exporter, log_exporter = _logger_with_events(enable_events=True) + payload = _payload( + status="failure", + error_information={ + "error_class": "RateLimitError", + "error_message": "429 rate limited", + "traceback": "Traceback (most recent call last) ...", + }, + ) + _emit_llm(logger, _kwargs(payload=payload), fail=True) + + (span,) = span_exporter.get_finished_spans() + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.attributes["event.name"] == GenAIEvent.OPERATION_EXCEPTION + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "429 rate limited" + assert record.attributes[ExceptionEvent.STACKTRACE] == "Traceback (most recent call last) ..." + assert record.trace_id == span.context.trace_id + assert record.span_id == span.context.span_id + + +def test_events_off_by_default_records_no_log_event_on_failure(): + """``enable_events`` defaults to off: even with a logs pipeline injected, a + failure records only the span-side error surface, no log event.""" + logger, span_exporter, log_exporter = _logger_with_events(enable_events=False) + payload = _payload( + status="failure", + error_information={"error_class": "RateLimitError", "error_message": "429"}, + ) + _emit_llm(logger, _kwargs(payload=payload), fail=True) + + assert len(span_exporter.get_finished_spans()) == 1 + assert log_exporter.get_finished_logs() == () + assert OpenTelemetryV2Config(exporter="in_memory").enable_events is False + + def test_sync_log_event_is_noop(): """V2 closes the span async-only; the sync callback runs out-of-context, so it no-ops (the span stays open on the carrier until the async callback)."""