diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 6b673967427..20cdf9662e1 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -240,7 +240,7 @@ class OpenTelemetryV2(CustomLogger): provider: Final = resolve_logger_provider(self.config, logger_provider) if provider is None: return None - return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME)) + return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource) # ====================================================================== # # Proxy global registration diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index d3628005bac..f552ba37655 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -245,6 +245,7 @@ class GenAIEvent: details, unlike the deprecated ``error.message`` span attribute. """ + NAME_KEY: Final = "event.name" OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception" diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py index e7b8e22ddcd..e95d31f886f 100644 --- a/litellm/integrations/otel/plumbing/events.py +++ b/litellm/integrations/otel/plumbing/events.py @@ -9,18 +9,28 @@ emitting that event; the exporter pipeline it rides is built in """ from dataclasses import dataclass +from time import time_ns from typing import Final -from opentelemetry._events import Event, EventLogger +from opentelemetry._logs import Logger, LogRecord from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.sdk.resources import Resource from opentelemetry.trace import SpanContext from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent +try: + from opentelemetry.sdk._logs import LogRecord as _SDKLogRecord +except ImportError: + _SDKLogRecord = None + +SDK_LOG_RECORD: Final[type[LogRecord] | None] = _SDKLogRecord + @dataclass(frozen=True, slots=True) class GenAIEventRecorder: - event_logger: EventLogger + event_logger: Logger + resource: Resource | None = None def record_operation_exception( self, @@ -30,24 +40,36 @@ class GenAIEventRecorder: stack_trace: str | None, timestamp_ns: int | None, ) -> None: - # ``exception.type`` and ``exception.message`` are the semconv-required - # pair and always ride the event; only the recommended stacktrace is - # conditional on the payload carrying one. stacktrace: Final = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else () - self.event_logger.emit( - Event( - name=GenAIEvent.OPERATION_EXCEPTION, - timestamp=timestamp_ns, + attributes: Final = dict( + ( + (GenAIEvent.NAME_KEY, GenAIEvent.OPERATION_EXCEPTION), + (ExceptionEvent.TYPE, error_type), + (ExceptionEvent.MESSAGE, message), + *stacktrace, + ) + ) + record: Final[LogRecord] = ( + SDK_LOG_RECORD( + timestamp=timestamp_ns or time_ns(), trace_id=span_context.trace_id, span_id=span_context.span_id, trace_flags=span_context.trace_flags, severity_number=SeverityNumber.WARN, - attributes=dict( - ( - (ExceptionEvent.TYPE, error_type), - (ExceptionEvent.MESSAGE, message), - *stacktrace, - ) - ), + body=message, + attributes=attributes, + resource=self.resource, # pyright: ignore[reportCallIssue] # SDK-only kwarg absent from the API LogRecord signature on the pin + ) + if SDK_LOG_RECORD is not None + else LogRecord( + timestamp=timestamp_ns or time_ns(), + trace_id=span_context.trace_id, + span_id=span_context.span_id, + trace_flags=span_context.trace_flags, + severity_number=SeverityNumber.WARN, + body=message, + attributes=attributes, + event_name=GenAIEvent.OPERATION_EXCEPTION, # pyright: ignore[reportCallIssue] # kwarg exists only on OTel 1.38+, absent from the pinned API signature ) ) + self.event_logger.emit(record) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index d52736a1303..e3474edaf14 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -9,11 +9,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics, trace -from opentelemetry._events import EventLogger -from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider +from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context from opentelemetry.metrics import MeterProvider, NoOpMeterProvider -from opentelemetry.sdk._events import EventLoggerProvider from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider from opentelemetry.sdk._logs.export import ( BatchLogRecordProcessor, @@ -1042,8 +1040,8 @@ def resolve_logger_provider( return provider -def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger: - return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version) +def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> Logger: + return provider.get_logger(name, litellm_version) def build_meter_provider( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 79747ac9956..07705e17d9a 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -1290,6 +1290,50 @@ def test_operation_exception_log_event_always_carries_required_pair(): assert ExceptionEvent.STACKTRACE not in attributes +def test_operation_exception_log_event_records_without_the_events_api(): + """Recording must not import the Events API modules (removed upstream in 1.44.0); + the SDK record path still exports.""" + import importlib + import sys + from unittest.mock import patch + + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs.export import InMemoryLogExporter + from opentelemetry.trace import INVALID_SPAN_CONTEXT + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + plumbing = ("litellm.integrations.otel.plumbing.events", "litellm.integrations.otel.plumbing.providers") + without_events_api = { + **{name: module for name, module in sys.modules.items() if name not in plumbing}, + "opentelemetry._events": None, + "opentelemetry.sdk._events": None, + } + with patch.dict(sys.modules, without_events_api, clear=True): + events_mod = importlib.import_module(plumbing[0]) + providers_mod = importlib.import_module(plumbing[1]) + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers_mod.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = events_mod.GenAIEventRecorder(providers_mod.get_event_logger(logger_provider)) + recorder.record_operation_exception( + span_context=INVALID_SPAN_CONTEXT, + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.attributes[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited" + assert record.severity_number == SeverityNumber.WARN + assert record.timestamp is not None + + def test_operation_exception_log_event_not_emitted_on_success(): engine, span_exporter, log_exporter = _engine_with_event_recorder() engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) @@ -1425,6 +1469,87 @@ def test_genai_mapper_guardrail_cost_in_spend_attr(): assert LiteLLM.GUARDRAIL_COST_IN_SPEND not in GenAIMapper().map(GuardrailSpanData.from_logging_entry(billed)) +def _sampled_span_context(): + from opentelemetry.trace import SpanContext, TraceFlags, TraceState + + return SpanContext( + trace_id=0x0AF7651916CD43DD8448EB211C80319C, + span_id=0x00F067AA0BA902B7, + is_remote=False, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + trace_state=TraceState(), + ) + + +def test_operation_exception_log_event_exports_through_console_exporter(): + """The emitted record serializes through a real SDK exporter: the console + exporter only handles SDK-shaped records (``to_json`` plus a resource), so + an API-shaped record crashed the export under the repo's pinned OTel.""" + import io + import json as json_mod + + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import ConsoleLogExporter, SimpleLogRecordProcessor + from opentelemetry.sdk.resources import Resource + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + out = io.StringIO() + logger_provider = LoggerProvider(resource=Resource.create({"service.name": "otel-event-test"})) + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(ConsoleLogExporter(out=out))) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + exported = json_mod.loads(out.getvalue()) + assert exported["attributes"][GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert exported["attributes"][ExceptionEvent.TYPE] == "RateLimitError" + assert exported["attributes"][ExceptionEvent.MESSAGE] == "rate limited" + assert exported["body"] == "rate limited" + assert exported["resource"]["attributes"]["service.name"] == "otel-event-test" + + +def test_operation_exception_log_event_encodes_for_otlp(): + """The OTLP log encoder reads ``log_record.resource`` and rejects a None + body on the pinned OTel line, so the event must encode into a real + ExportLogsServiceRequest, not only land in an in-memory exporter.""" + from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + from litellm.integrations.otel.model.semconv import GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + request = encode_logs(log_exporter.get_finished_logs()) + (resource_logs,) = request.resource_logs + (scope_logs,) = resource_logs.scope_logs + (encoded,) = scope_logs.log_records + encoded_attrs = {a.key: a.value.string_value for a in encoded.attributes} + assert encoded_attrs[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert encoded.body.string_value == "rate limited" + resource_attrs = {a.key: a.value.string_value for a in resource_logs.resource.attributes} + assert resource_attrs["service.name"] == logger_provider.resource.attributes["service.name"] + + + + def _isolate_v2_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None: for key in ( "SSL_VERIFY",