mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(otel): record the GenAI exception event through the Logs API on both OpenTelemetry lines (#42431)
* fix(otel): record the GenAI exception event without the removed Events API OpenTelemetry removed opentelemetry._events in 1.44.0, so the three imports of it broke 7 modules under litellm.integrations.otel, including the entry point. Two things then failed quietly: with LITELLM_OTEL_V2 set the otel callback resolved to None and nothing was exported, and with it unset the newrelic callback was dropped as well, because that branch imports the v2 logger ungated Build and emit the event through the Logs API, which both lines carry. The event name keeps riding the event.name attribute: the event_name log record field that replaces it only exists from 1.44.0, and this package pins 1.28.0, so the attribute is the only form both can write. It is also what the Events API wrote, so exported events keep their shape Emitting a plain record drops the default the Events SDK applied, so the timestamp now falls back to time_ns() here * style(otel): trim the event name key and regression test prose Keep only the constraint a reader cannot infer from the code, that the event_name record field does not exist on the pinned OpenTelemetry line * fix(otel): export the GenAI exception event on both OpenTelemetry 1.28 and 1.44 lines Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(otel): drop the record selection comment Co-authored-by: Pawan-Shahane <shahanepawan511@gmail.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): collapse the record selection conditional for ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): build the record fields with a dict literal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): set the native event_name on the 1.44 log record Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): spell out the record kwargs so the type gate sees each call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): suppress the version-gated kwargs for the type gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(otel): drop the version-window prose and correct the event_name suppression reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): wrap the compat test docstring to the line limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Pawan-Shahane <shahanepawan511@gmail.com> Co-authored-by: yucheng <yucheng@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
95d4613bd3
commit
238f434153
5 changed files with 168 additions and 22 deletions
|
|
@ -240,7 +240,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
provider: Final = resolve_logger_provider(self.config, logger_provider)
|
||||
if provider is None:
|
||||
return None
|
||||
return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME))
|
||||
return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource)
|
||||
|
||||
# ====================================================================== #
|
||||
# Proxy global registration
|
||||
|
|
|
|||
|
|
@ -245,6 +245,7 @@ class GenAIEvent:
|
|||
details, unlike the deprecated ``error.message`` span attribute.
|
||||
"""
|
||||
|
||||
NAME_KEY: Final = "event.name"
|
||||
OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,18 +9,28 @@ emitting that event; the exporter pipeline it rides is built in
|
|||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from time import time_ns
|
||||
from typing import Final
|
||||
|
||||
from opentelemetry._events import Event, EventLogger
|
||||
from opentelemetry._logs import Logger, LogRecord
|
||||
from opentelemetry._logs.severity import SeverityNumber
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.trace import SpanContext
|
||||
|
||||
from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent
|
||||
|
||||
try:
|
||||
from opentelemetry.sdk._logs import LogRecord as _SDKLogRecord
|
||||
except ImportError:
|
||||
_SDKLogRecord = None
|
||||
|
||||
SDK_LOG_RECORD: Final[type[LogRecord] | None] = _SDKLogRecord
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GenAIEventRecorder:
|
||||
event_logger: EventLogger
|
||||
event_logger: Logger
|
||||
resource: Resource | None = None
|
||||
|
||||
def record_operation_exception(
|
||||
self,
|
||||
|
|
@ -30,24 +40,36 @@ class GenAIEventRecorder:
|
|||
stack_trace: str | None,
|
||||
timestamp_ns: int | None,
|
||||
) -> None:
|
||||
# ``exception.type`` and ``exception.message`` are the semconv-required
|
||||
# pair and always ride the event; only the recommended stacktrace is
|
||||
# conditional on the payload carrying one.
|
||||
stacktrace: Final = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else ()
|
||||
self.event_logger.emit(
|
||||
Event(
|
||||
name=GenAIEvent.OPERATION_EXCEPTION,
|
||||
timestamp=timestamp_ns,
|
||||
attributes: Final = dict(
|
||||
(
|
||||
(GenAIEvent.NAME_KEY, GenAIEvent.OPERATION_EXCEPTION),
|
||||
(ExceptionEvent.TYPE, error_type),
|
||||
(ExceptionEvent.MESSAGE, message),
|
||||
*stacktrace,
|
||||
)
|
||||
)
|
||||
record: Final[LogRecord] = (
|
||||
SDK_LOG_RECORD(
|
||||
timestamp=timestamp_ns or time_ns(),
|
||||
trace_id=span_context.trace_id,
|
||||
span_id=span_context.span_id,
|
||||
trace_flags=span_context.trace_flags,
|
||||
severity_number=SeverityNumber.WARN,
|
||||
attributes=dict(
|
||||
(
|
||||
(ExceptionEvent.TYPE, error_type),
|
||||
(ExceptionEvent.MESSAGE, message),
|
||||
*stacktrace,
|
||||
)
|
||||
),
|
||||
body=message,
|
||||
attributes=attributes,
|
||||
resource=self.resource, # pyright: ignore[reportCallIssue] # SDK-only kwarg absent from the API LogRecord signature on the pin
|
||||
)
|
||||
if SDK_LOG_RECORD is not None
|
||||
else LogRecord(
|
||||
timestamp=timestamp_ns or time_ns(),
|
||||
trace_id=span_context.trace_id,
|
||||
span_id=span_context.span_id,
|
||||
trace_flags=span_context.trace_flags,
|
||||
severity_number=SeverityNumber.WARN,
|
||||
body=message,
|
||||
attributes=attributes,
|
||||
event_name=GenAIEvent.OPERATION_EXCEPTION, # pyright: ignore[reportCallIssue] # kwarg exists only on OTel 1.38+, absent from the pinned API signature
|
||||
)
|
||||
)
|
||||
self.event_logger.emit(record)
|
||||
|
|
|
|||
|
|
@ -9,11 +9,9 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
from opentelemetry import _logs, baggage, metrics, trace
|
||||
from opentelemetry._events import EventLogger
|
||||
from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider
|
||||
from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.metrics import MeterProvider, NoOpMeterProvider
|
||||
from opentelemetry.sdk._events import EventLoggerProvider
|
||||
from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider
|
||||
from opentelemetry.sdk._logs.export import (
|
||||
BatchLogRecordProcessor,
|
||||
|
|
@ -1042,8 +1040,8 @@ def resolve_logger_provider(
|
|||
return provider
|
||||
|
||||
|
||||
def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger:
|
||||
return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version)
|
||||
def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> Logger:
|
||||
return provider.get_logger(name, litellm_version)
|
||||
|
||||
|
||||
def build_meter_provider(
|
||||
|
|
|
|||
|
|
@ -1290,6 +1290,50 @@ def test_operation_exception_log_event_always_carries_required_pair():
|
|||
assert ExceptionEvent.STACKTRACE not in attributes
|
||||
|
||||
|
||||
def test_operation_exception_log_event_records_without_the_events_api():
|
||||
"""Recording must not import the Events API modules (removed upstream in 1.44.0);
|
||||
the SDK record path still exports."""
|
||||
import importlib
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
from opentelemetry._logs.severity import SeverityNumber
|
||||
from opentelemetry.sdk._logs.export import InMemoryLogExporter
|
||||
from opentelemetry.trace import INVALID_SPAN_CONTEXT
|
||||
|
||||
from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent
|
||||
|
||||
plumbing = ("litellm.integrations.otel.plumbing.events", "litellm.integrations.otel.plumbing.providers")
|
||||
without_events_api = {
|
||||
**{name: module for name, module in sys.modules.items() if name not in plumbing},
|
||||
"opentelemetry._events": None,
|
||||
"opentelemetry.sdk._events": None,
|
||||
}
|
||||
with patch.dict(sys.modules, without_events_api, clear=True):
|
||||
events_mod = importlib.import_module(plumbing[0])
|
||||
providers_mod = importlib.import_module(plumbing[1])
|
||||
|
||||
log_exporter = InMemoryLogExporter()
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True)
|
||||
logger_provider = providers_mod.build_logger_provider(cfg, log_exporter=log_exporter)
|
||||
recorder = events_mod.GenAIEventRecorder(providers_mod.get_event_logger(logger_provider))
|
||||
recorder.record_operation_exception(
|
||||
span_context=INVALID_SPAN_CONTEXT,
|
||||
error_type="RateLimitError",
|
||||
message="rate limited",
|
||||
stack_trace=None,
|
||||
timestamp_ns=None,
|
||||
)
|
||||
|
||||
(log,) = log_exporter.get_finished_logs()
|
||||
record = log.log_record
|
||||
assert record.attributes[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION
|
||||
assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError"
|
||||
assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited"
|
||||
assert record.severity_number == SeverityNumber.WARN
|
||||
assert record.timestamp is not None
|
||||
|
||||
|
||||
def test_operation_exception_log_event_not_emitted_on_success():
|
||||
engine, span_exporter, log_exporter = _engine_with_event_recorder()
|
||||
engine.emit(SpanRole.LLM_CALL, _llm_call_data(None))
|
||||
|
|
@ -1425,6 +1469,87 @@ def test_genai_mapper_guardrail_cost_in_spend_attr():
|
|||
assert LiteLLM.GUARDRAIL_COST_IN_SPEND not in GenAIMapper().map(GuardrailSpanData.from_logging_entry(billed))
|
||||
|
||||
|
||||
def _sampled_span_context():
|
||||
from opentelemetry.trace import SpanContext, TraceFlags, TraceState
|
||||
|
||||
return SpanContext(
|
||||
trace_id=0x0AF7651916CD43DD8448EB211C80319C,
|
||||
span_id=0x00F067AA0BA902B7,
|
||||
is_remote=False,
|
||||
trace_flags=TraceFlags(TraceFlags.SAMPLED),
|
||||
trace_state=TraceState(),
|
||||
)
|
||||
|
||||
|
||||
def test_operation_exception_log_event_exports_through_console_exporter():
|
||||
"""The emitted record serializes through a real SDK exporter: the console
|
||||
exporter only handles SDK-shaped records (``to_json`` plus a resource), so
|
||||
an API-shaped record crashed the export under the repo's pinned OTel."""
|
||||
import io
|
||||
import json as json_mod
|
||||
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import ConsoleLogExporter, SimpleLogRecordProcessor
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
|
||||
from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent
|
||||
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
|
||||
|
||||
out = io.StringIO()
|
||||
logger_provider = LoggerProvider(resource=Resource.create({"service.name": "otel-event-test"}))
|
||||
logger_provider.add_log_record_processor(SimpleLogRecordProcessor(ConsoleLogExporter(out=out)))
|
||||
recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource)
|
||||
recorder.record_operation_exception(
|
||||
span_context=_sampled_span_context(),
|
||||
error_type="RateLimitError",
|
||||
message="rate limited",
|
||||
stack_trace=None,
|
||||
timestamp_ns=None,
|
||||
)
|
||||
|
||||
exported = json_mod.loads(out.getvalue())
|
||||
assert exported["attributes"][GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION
|
||||
assert exported["attributes"][ExceptionEvent.TYPE] == "RateLimitError"
|
||||
assert exported["attributes"][ExceptionEvent.MESSAGE] == "rate limited"
|
||||
assert exported["body"] == "rate limited"
|
||||
assert exported["resource"]["attributes"]["service.name"] == "otel-event-test"
|
||||
|
||||
|
||||
def test_operation_exception_log_event_encodes_for_otlp():
|
||||
"""The OTLP log encoder reads ``log_record.resource`` and rejects a None
|
||||
body on the pinned OTel line, so the event must encode into a real
|
||||
ExportLogsServiceRequest, not only land in an in-memory exporter."""
|
||||
from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs
|
||||
from opentelemetry.sdk._logs.export import InMemoryLogExporter
|
||||
|
||||
from litellm.integrations.otel.model.semconv import GenAIEvent
|
||||
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
|
||||
|
||||
log_exporter = InMemoryLogExporter()
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True)
|
||||
logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter)
|
||||
recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource)
|
||||
recorder.record_operation_exception(
|
||||
span_context=_sampled_span_context(),
|
||||
error_type="RateLimitError",
|
||||
message="rate limited",
|
||||
stack_trace=None,
|
||||
timestamp_ns=None,
|
||||
)
|
||||
|
||||
request = encode_logs(log_exporter.get_finished_logs())
|
||||
(resource_logs,) = request.resource_logs
|
||||
(scope_logs,) = resource_logs.scope_logs
|
||||
(encoded,) = scope_logs.log_records
|
||||
encoded_attrs = {a.key: a.value.string_value for a in encoded.attributes}
|
||||
assert encoded_attrs[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION
|
||||
assert encoded.body.string_value == "rate limited"
|
||||
resource_attrs = {a.key: a.value.string_value for a in resource_logs.resource.attributes}
|
||||
assert resource_attrs["service.name"] == logger_provider.resource.attributes["service.name"]
|
||||
|
||||
|
||||
|
||||
|
||||
def _isolate_v2_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for key in (
|
||||
"SSL_VERIFY",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue