From bf8c35e2d4f6a74759bc5c9cd3d8c0c9020b376e Mon Sep 17 00:00:00 2001 From: Yashwanth-Kumar-Kotla Date: Mon, 31 Aug 2026 15:40:52 -0500 Subject: [PATCH 1/2] fix(otel): don't cast None metadata values to empty-string span attributes user_api_key_budget_reset_at (and any other metadata field that's sometimes None, e.g. a key with no budget configured) was being cast to "" by _cast_as_primitive_value_type and set on the span like any other value. A field that sends two different shapes for the same attribute -- an ISO timestamp string sometimes, an empty string other times -- breaks backends that infer a field's type from the first value they see, such as OpenSearch's dynamic index mapping. Skip None values in the metadata attribute loop instead, matching the same pattern already used for cost_breakdown two loops below. Fixes #29583 --- litellm/integrations/opentelemetry.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index e8f3b305139..dc0cef16d66 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2361,7 +2361,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ############################################# metadata: Final = standard_logging_payload["metadata"] for key, value in metadata.items(): - self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value) + # Skip None values instead of letting safe_set_attribute cast them to "". + # Fields that are sometimes a real value (e.g. an ISO timestamp string + # like user_api_key_budget_reset_at) and sometimes None would otherwise + # send two different shapes for the same attribute, and backends that + # infer a field's type from the first value they see (e.g. OpenSearch's + # dynamic mapping) lock onto whichever shape arrives first -- causing the + # other shape to fail once the mapping is set. See issue #29583. + if value is not None: + self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value) # get hidden params hidden_params: Final = getattr(standard_logging_payload, "hidden_params", None) or ( From d010ed42672bd8574947b7ee225f926cce749511 Mon Sep 17 00:00:00 2001 From: Yashwanth-Kumar-Kotla Date: Mon, 31 Aug 2026 15:41:07 -0500 Subject: [PATCH 2/2] test(otel): regression test for None metadata values on spans Covers issue #29583: a None-valued metadata field (e.g. user_api_key_budget_reset_at with no budget configured) should be omitted from the span, not sent as an empty string. Also asserts a real timestamp value for the same field is still set correctly, so the fix doesn't just suppress the field entirely. Verified against a control: this test fails on the pre-fix code and passes with it. --- .../integrations/test_opentelemetry.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 9ec8489f784..a237a15c754 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -391,6 +391,86 @@ class TestOpenTelemetryCostBreakdown(unittest.TestCase): assert ("gen_ai.cost.original_cost", 0.004) not in call_args_list +class TestOpenTelemetryMetadataNoneValues(unittest.TestCase): + """ + Regression test for https://github.com/BerriAI/litellm/issues/29583 + + None-valued metadata fields (e.g. user_api_key_budget_reset_at for a key + with no budget configured) were being cast to "" and set on the span like + any other value. A field that is sometimes a real value and sometimes "" + sends two different shapes for the same attribute -- backends that infer + a field's type from the first value they see (e.g. OpenSearch's dynamic + mapping) lock onto whichever shape arrives first, so the other shape then + fails. None-valued metadata should be omitted from the span instead. + """ + + def test_none_metadata_value_is_not_set_on_span(self): + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": { + "user_api_key_budget_reset_at": None, + "user_api_key_team_id": "team-123", + }, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + call_args_list = [call[0] for call in mock_span.set_attribute.call_args_list] + set_keys = [args[0] for args in call_args_list] + + assert "metadata.user_api_key_budget_reset_at" not in set_keys + assert ("metadata.user_api_key_budget_reset_at", "") not in call_args_list + mock_span.set_attribute.assert_any_call("metadata.user_api_key_team_id", "team-123") + + def test_real_timestamp_metadata_value_is_still_set_on_span(self): + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": { + "user_api_key_budget_reset_at": "2026-09-15T00:00:00+00:00", + }, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + mock_span.set_attribute.assert_any_call( + "metadata.user_api_key_budget_reset_at", "2026-09-15T00:00:00+00:00" + ) + + class TestOpenTelemetryProviderInitialization(unittest.TestCase): """Test suite for verifying provider initialization respects existing providers"""