From 38166022751da1dc53f0b8cbe66bbe96685469b3 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:51:44 +1000 Subject: [PATCH 1/2] fix(otel): set a body on the GenAI exception event so OTLP can encode it `GenAIEventRecorder.record_operation_exception` built its `Event` without a `body`. `opentelemetry._events.Event.body` defaults to `None`, and OTLP's `AnyValue` cannot represent `None`, so `_encode_value` raises `Invalid type ` while serialising the batch. That is data loss rather than noise: `BatchLogRecordProcessor._export_batch` catches the exception, clears the records and returns them as exported, so one unencodable record discards the whole batch with no retry. Since this recorder is the only emit site on the v2 logs signal, every batch on that pipeline was made up entirely of unencodable records and nothing was ever delivered. A dependency bump does not reach it: the encoder grew a `None` branch in opentelemetry-exporter-otlp-proto-common 1.43.0, and litellm pins 1.28.0. Setting the body fixes it on every OTel version. The message is the natural body for a WARN record and adds no data the attributes don't already carry. The existing tests all use `InMemoryLogExporter`, which never encodes, which is why this passed CI. The new tests run the real OTLP encoder. --- litellm/integrations/otel/plumbing/events.py | 12 +++++ .../otel/test_otel_v2_components.py | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py index e7b8e22ddcd..bc89650d40d 100644 --- a/litellm/integrations/otel/plumbing/events.py +++ b/litellm/integrations/otel/plumbing/events.py @@ -37,6 +37,18 @@ class GenAIEventRecorder: self.event_logger.emit( Event( name=GenAIEvent.OPERATION_EXCEPTION, + # The body MUST be set. ``Event.body`` defaults to ``None``, and + # OTLP's ``AnyValue`` has no representation for ``None``: the + # exporter's ``_encode_value`` raises ``Invalid type ``, and ``BatchLogRecordProcessor._export_batch`` + # swallows that and discards the WHOLE batch — so one such event + # silently destroys every log record batched with it. The + # encoder grew a ``None`` branch in opentelemetry-exporter-otlp- + # proto-common 1.43.0, but litellm pins 1.28.0, so this event is + # unexportable as shipped. The message is the natural body for a + # WARN record and adds no data the attributes don't already + # carry. + body=message, timestamp=timestamp_ns, trace_id=span_context.trace_id, span_id=span_context.span_id, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index d856d6871a3..429795477ac 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -940,6 +940,51 @@ def test_operation_exception_log_event_always_carries_required_pair(): assert ExceptionEvent.STACKTRACE not in attributes +def test_operation_exception_log_event_is_otlp_encodable(): + """The event must survive the real OTLP encoder, not just an in-memory exporter. + + ``Event.body`` defaults to ``None`` and OTLP's ``AnyValue`` cannot represent + it, so an event emitted without a body raises ``Invalid type `` inside ``encode_logs``. That happens on the exporter's batch + thread, where ``BatchLogRecordProcessor._export_batch`` catches it, logs it + and drops the WHOLE batch — every record batched alongside is lost silently. + Every other test here uses ``InMemoryLogExporter``, which never encodes, so + only an explicit encode step can catch this. + """ + encode_logs = pytest.importorskip( + "opentelemetry.exporter.otlp.proto.common._internal._log_encoder" + ).encode_logs + + engine, _, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.LLM_CALL, + _llm_call_data(SpanError(error_type="RateLimitError", message="rate limited")), + ) + logs = log_exporter.get_finished_logs() + + # Raises "Invalid type of value None" when body is unset. + encode_logs(logs).SerializeToString() + + +def test_operation_exception_log_event_body_is_never_none(): + """A batch is dropped whole on an unencodable body, so this is asserted for + both event shapes: with and without a stacktrace.""" + engine, _, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.LLM_CALL, + _llm_call_data(SpanError(error_type="APIError", message="boom")), + ) + engine.emit( + SpanRole.LLM_CALL, + _llm_call_data( + SpanError(error_type="APIError", message="boom", stack_trace="Traceback ...") + ), + ) + + bodies = [log.log_record.body for log in log_exporter.get_finished_logs()] + assert bodies == ["boom", "boom"] + + def test_operation_exception_log_event_not_emitted_on_success(): engine, span_exporter, log_exporter = _engine_with_event_recorder() engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) From 15cfe744d7da3ff3f738f8929beedc9d4321f7c3 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:46:22 +1000 Subject: [PATCH 2/2] test(otel): assert the encoded exception event carries the message body --- .../integrations/otel/test_otel_v2_components.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 429795477ac..5b1defd87fc 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -963,7 +963,13 @@ def test_operation_exception_log_event_is_otlp_encodable(): logs = log_exporter.get_finished_logs() # Raises "Invalid type of value None" when body is unset. - encode_logs(logs).SerializeToString() + request = encode_logs(logs) + assert request.SerializeToString() + + (resource_logs,) = request.resource_logs + (scope_logs,) = resource_logs.scope_logs + (log_record,) = scope_logs.log_records + assert log_record.body.string_value == "rate limited" def test_operation_exception_log_event_body_is_never_none():