From 2cc5033532e1b307d7d3074cd53e9b31e3b5a13f Mon Sep 17 00:00:00 2001 From: singularity-14 Date: Thu, 1 Oct 2026 09:23:12 +0530 Subject: [PATCH 1/2] fix(otel): record sampling params explicitly set to zero on spans The OTel and Arize integrations gated gen_ai.request.temperature, top_p and max_tokens on truthiness, so a request that explicitly set any of them to 0 produced a span with the attribute missing entirely. temperature=0 is the common case for deterministic output, which made it indistinguishable from never setting temperature at all. Gate on `is not None` instead, matching what the semconv path in the same module and the langtrace and traceloop integrations already do. Verified against a live proxy: with temperature=0 and max_tokens=16 in one request, the non-zero max_tokens was recorded before and after while the zero temperature only appears after the change. --- litellm/integrations/arize/_utils.py | 14 ++++--- litellm/integrations/opentelemetry.py | 31 ++++---------- .../integrations/arize/test_arize_utils.py | 38 +++++++++++++++++ tests/unit/integrations/test_opentelemetry.py | 41 +++++++++++++++++++ 4 files changed, 95 insertions(+), 29 deletions(-) diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 0271cf1e03c..4f1f3b1c25b 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -529,12 +529,14 @@ def _set_request_attributes( litellm_params.get("custom_llm_provider", "Unknown"), ) - if optional_params.get("max_tokens"): - safe_set_attribute(span, "llm.request.max_tokens", optional_params.get("max_tokens")) - if optional_params.get("temperature"): - safe_set_attribute(span, "llm.request.temperature", optional_params.get("temperature")) - if optional_params.get("top_p"): - safe_set_attribute(span, "llm.request.top_p", optional_params.get("top_p")) + for source_key, request_attribute in ( + ("max_tokens", "llm.request.max_tokens"), + ("temperature", "llm.request.temperature"), + ("top_p", "llm.request.top_p"), + ): + request_value = optional_params.get(source_key) + if request_value is not None: + safe_set_attribute(span, request_attribute, request_value) safe_set_attribute(span, "llm.is_streaming", str(optional_params.get("stream", False))) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 8d588896b2f..0630adba202 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2469,29 +2469,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=provider_name, ) - # The maximum number of tokens the LLM generates for a request. - if optional_params.get("max_tokens"): - self.safe_set_attribute( - span=span, - key=SpanAttributes.LLM_REQUEST_MAX_TOKENS.value, - value=optional_params.get("max_tokens"), - ) - - # The temperature setting for the LLM request. - if optional_params.get("temperature"): - self.safe_set_attribute( - span=span, - key=SpanAttributes.LLM_REQUEST_TEMPERATURE.value, - value=optional_params.get("temperature"), - ) - - # The top_p sampling setting for the LLM request. - if optional_params.get("top_p"): - self.safe_set_attribute( - span=span, - key=SpanAttributes.LLM_REQUEST_TOP_P.value, - value=optional_params.get("top_p"), - ) + for source_key, request_attribute in ( + ("max_tokens", SpanAttributes.LLM_REQUEST_MAX_TOKENS.value), + ("temperature", SpanAttributes.LLM_REQUEST_TEMPERATURE.value), + ("top_p", SpanAttributes.LLM_REQUEST_TOP_P.value), + ): + request_value = optional_params.get(source_key) + if request_value is not None: + self.safe_set_attribute(span=span, key=request_attribute, value=request_value) if self._gen_ai_semconv_latest_experimental: # Semconv emits gen_ai.request.stream (only when streaming) via diff --git a/tests/unit/integrations/arize/test_arize_utils.py b/tests/unit/integrations/arize/test_arize_utils.py index 167b083e147..3ee723d439b 100644 --- a/tests/unit/integrations/arize/test_arize_utils.py +++ b/tests/unit/integrations/arize/test_arize_utils.py @@ -181,6 +181,44 @@ def test_arize_set_attributes(): span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 40) +def test_arize_records_zero_valued_sampling_params(): + """A sampling param the caller set to 0 still has to reach the span. + + temperature=0 is what you send for deterministic output, so dropping it + hides the single most common setting from the trace. + """ + from unittest.mock import MagicMock + + from litellm.types.utils import ModelResponse + + span = MagicMock() + + kwargs = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Basic Request Content"}], + "standard_logging_object": { + "model_parameters": {}, + "metadata": {}, + "call_type": "completion", + }, + "optional_params": {"max_tokens": 0, "temperature": 0, "top_p": 0}, + "litellm_params": {"custom_llm_provider": "openai"}, + } + + response_obj = ModelResponse( + usage={"total_tokens": 100, "completion_tokens": 60, "prompt_tokens": 40}, + choices=[Choices(message={"role": "assistant", "content": "Basic Response Content"})], + model="gpt-4o", + id="chatcmpl-ID", + ) + + ArizeLogger.set_arize_attributes(span, kwargs, response_obj) + + span.set_attribute.assert_any_call("llm.request.temperature", 0) + span.set_attribute.assert_any_call("llm.request.top_p", 0) + span.set_attribute.assert_any_call("llm.request.max_tokens", 0) + + def test_arize_set_attributes_responses_api(): """ Test setting attributes for Responses API with mixed output (reasoning + message). diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 175bd95c263..86c7b783431 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -402,6 +402,47 @@ class TestOpenTelemetryCostBreakdown(unittest.TestCase): assert ("gen_ai.cost.original_cost", 0.004) not in call_args_list +class TestOpenTelemetryZeroValuedRequestParams(unittest.TestCase): + def test_zero_valued_sampling_params_reach_the_span(self): + """A sampling param the caller set to 0 still has to reach the span. + + temperature=0 is what you send for deterministic output, so dropping it + hides the single most common setting from the trace. The semconv path + already records these, so the legacy path has to agree. + """ + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {"max_tokens": 0, "temperature": 0, "top_p": 0}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + from litellm.proxy._types import SpanAttributes + + recorded = dict(call[0] for call in mock_span.set_attribute.call_args_list if len(call[0]) == 2) + + assert recorded.get(SpanAttributes.LLM_REQUEST_TEMPERATURE.value) == 0 + assert recorded.get(SpanAttributes.LLM_REQUEST_TOP_P.value) == 0 + assert recorded.get(SpanAttributes.LLM_REQUEST_MAX_TOKENS.value) == 0 + + class TestOpenTelemetryProviderInitialization(unittest.TestCase): """Test suite for verifying provider initialization respects existing providers""" From cdae4e044ea6725082da4ddd67ff043e1efe3843 Mon Sep 17 00:00:00 2001 From: singularity-14 Date: Mon, 5 Oct 2026 22:24:56 +0530 Subject: [PATCH 2/2] fix(otel): read max_tokens from max_completion_tokens as a fallback get_optional_params leaves max_completion_tokens under its own key rather than folding it into max_tokens, so a request that sends it produced a span with no gen_ai.request.max_tokens or llm.request.max_tokens at all. Resolve the two the same way LLMRequestParams.from_model_parameters already does in litellm/integrations/otel/model/payloads.py: prefer max_tokens when it is present, including when it is 0, and fall back to max_completion_tokens otherwise. Reported by @chrikrah while reviewing #43977. --- litellm/integrations/arize/_utils.py | 16 ++++--- litellm/integrations/opentelemetry.py | 16 ++++--- .../integrations/arize/test_arize_utils.py | 44 +++++++++++++++++++ tests/unit/integrations/test_opentelemetry.py | 43 ++++++++++++++++++ 4 files changed, 105 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 4f1f3b1c25b..95237136b7b 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -529,14 +529,16 @@ def _set_request_attributes( litellm_params.get("custom_llm_provider", "Unknown"), ) - for source_key, request_attribute in ( - ("max_tokens", "llm.request.max_tokens"), - ("temperature", "llm.request.temperature"), - ("top_p", "llm.request.top_p"), + for source_keys, request_attribute in ( + (("max_tokens", "max_completion_tokens"), "llm.request.max_tokens"), + (("temperature",), "llm.request.temperature"), + (("top_p",), "llm.request.top_p"), ): - request_value = optional_params.get(source_key) - if request_value is not None: - safe_set_attribute(span, request_attribute, request_value) + for source_key in source_keys: + request_value = optional_params.get(source_key) + if request_value is not None: + safe_set_attribute(span, request_attribute, request_value) + break safe_set_attribute(span, "llm.is_streaming", str(optional_params.get("stream", False))) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0630adba202..998bf48cd86 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2469,14 +2469,16 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=provider_name, ) - for source_key, request_attribute in ( - ("max_tokens", SpanAttributes.LLM_REQUEST_MAX_TOKENS.value), - ("temperature", SpanAttributes.LLM_REQUEST_TEMPERATURE.value), - ("top_p", SpanAttributes.LLM_REQUEST_TOP_P.value), + for source_keys, request_attribute in ( + (("max_tokens", "max_completion_tokens"), SpanAttributes.LLM_REQUEST_MAX_TOKENS.value), + (("temperature",), SpanAttributes.LLM_REQUEST_TEMPERATURE.value), + (("top_p",), SpanAttributes.LLM_REQUEST_TOP_P.value), ): - request_value = optional_params.get(source_key) - if request_value is not None: - self.safe_set_attribute(span=span, key=request_attribute, value=request_value) + for source_key in source_keys: + request_value = optional_params.get(source_key) + if request_value is not None: + self.safe_set_attribute(span=span, key=request_attribute, value=request_value) + break if self._gen_ai_semconv_latest_experimental: # Semconv emits gen_ai.request.stream (only when streaming) via diff --git a/tests/unit/integrations/arize/test_arize_utils.py b/tests/unit/integrations/arize/test_arize_utils.py index 3ee723d439b..478a8321ede 100644 --- a/tests/unit/integrations/arize/test_arize_utils.py +++ b/tests/unit/integrations/arize/test_arize_utils.py @@ -219,6 +219,50 @@ def test_arize_records_zero_valued_sampling_params(): span.set_attribute.assert_any_call("llm.request.max_tokens", 0) +@pytest.mark.parametrize( + "optional_params, expected", + [ + ({"max_completion_tokens": 256}, 256), + ({"max_tokens": 128}, 128), + ({"max_tokens": 128, "max_completion_tokens": 256}, 128), + ({"max_tokens": 0, "max_completion_tokens": 256}, 0), + ], +) +def test_arize_records_max_tokens_from_either_request_field(optional_params, expected): + """Callers send max_completion_tokens instead of max_tokens, so the span has to + read both. max_tokens wins when present, including when it is 0, which matches + how LLMRequestParams.from_model_parameters resolves the two. + """ + from unittest.mock import MagicMock + + from litellm.types.utils import ModelResponse + + span = MagicMock() + + kwargs = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Basic Request Content"}], + "standard_logging_object": { + "model_parameters": {}, + "metadata": {}, + "call_type": "completion", + }, + "optional_params": optional_params, + "litellm_params": {"custom_llm_provider": "openai"}, + } + + response_obj = ModelResponse( + usage={"total_tokens": 100, "completion_tokens": 60, "prompt_tokens": 40}, + choices=[Choices(message={"role": "assistant", "content": "Basic Response Content"})], + model="gpt-4o", + id="chatcmpl-ID", + ) + + ArizeLogger.set_arize_attributes(span, kwargs, response_obj) + + span.set_attribute.assert_any_call("llm.request.max_tokens", expected) + + def test_arize_set_attributes_responses_api(): """ Test setting attributes for Responses API with mixed output (reasoning + message). diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 86c7b783431..16cc77ae815 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -442,6 +442,49 @@ class TestOpenTelemetryZeroValuedRequestParams(unittest.TestCase): assert recorded.get(SpanAttributes.LLM_REQUEST_TOP_P.value) == 0 assert recorded.get(SpanAttributes.LLM_REQUEST_MAX_TOKENS.value) == 0 + @parameterized.expand( + [ + ({"max_completion_tokens": 256}, 256), + ({"max_tokens": 128}, 128), + ({"max_tokens": 128, "max_completion_tokens": 256}, 128), + ({"max_tokens": 0, "max_completion_tokens": 256}, 0), + ] + ) + def test_max_tokens_is_read_from_either_request_field(self, optional_params, expected): + """Callers send max_completion_tokens instead of max_tokens, so the span has to + read both. max_tokens wins when present, including when it is 0, which matches + how LLMRequestParams.from_model_parameters resolves the two. + """ + from litellm.proxy._types import SpanAttributes + + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": optional_params, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + recorded = dict(call[0] for call in mock_span.set_attribute.call_args_list if len(call[0]) == 2) + + assert recorded.get(SpanAttributes.LLM_REQUEST_MAX_TOKENS.value) == expected + class TestOpenTelemetryProviderInitialization(unittest.TestCase): """Test suite for verifying provider initialization respects existing providers"""