diff --git a/litellm/constants.py b/litellm/constants.py index 765bbfe1e54..57a8fc2b97c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -49,6 +49,8 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = ( # Set to 0 to disable truncation. MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64)) REDACTED_BY_LITELLM: Final = "redacted-by-litellm" +# in-memory stand-in handed to provider converters for redacted arguments; never stored +REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}" MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index a93c45ef840..3d044c3ea15 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -10,6 +10,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import ( LangfuseLLMObsOTELAttributes, ) from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, ) @@ -197,7 +198,11 @@ class LangfuseOtelLogger(OpenTelemetry): ) elif item_type == "function_call": arguments_str = getattr(item, "arguments", "{}") - arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str + arguments_obj = ( + safe_json_loads(arguments_str, default={}) + if isinstance(arguments_str, str) + else arguments_str + ) langfuse_tool_call = { "id": getattr(item, "id", ""), "name": getattr(item, "name", ""), diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index be187b091b8..064228bf8d7 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -97,16 +97,18 @@ def _redact_function_call(function_call) -> None: def _redact_choice_content(choice): """Helper to redact content in a choice (message or delta).""" if isinstance(choice, litellm.Choices): - choice.message.content = REDACTED_BY_LITELLM - if hasattr(choice.message, "reasoning_content"): + if choice.message.content is not None: + choice.message.content = REDACTED_BY_LITELLM + if getattr(choice.message, "reasoning_content", None) is not None: choice.message.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.message, "thinking_blocks"): choice.message.thinking_blocks = None _redact_tool_calls(getattr(choice.message, "tool_calls", None)) _redact_function_call(getattr(choice.message, "function_call", None)) elif isinstance(choice, litellm.utils.StreamingChoices): - choice.delta.content = REDACTED_BY_LITELLM - if hasattr(choice.delta, "reasoning_content"): + if choice.delta.content is not None: + choice.delta.content = REDACTED_BY_LITELLM + if getattr(choice.delta, "reasoning_content", None) is not None: choice.delta.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.delta, "thinking_blocks"): choice.delta.thinking_blocks = None @@ -117,19 +119,19 @@ def _redact_choice_content(choice): def _redact_responses_api_output(output_items): """Helper to redact ResponsesAPIResponse output items.""" for output_item in output_items: - if hasattr(output_item, "text"): + if getattr(output_item, "text", None) is not None: output_item.text = REDACTED_BY_LITELLM if hasattr(output_item, "content") and isinstance(output_item.content, list): for content_part in output_item.content: - if hasattr(content_part, "text"): + if getattr(content_part, "text", None) is not None: content_part.text = REDACTED_BY_LITELLM # Redact reasoning items in output array if hasattr(output_item, "type") and output_item.type == "reasoning": if hasattr(output_item, "summary") and isinstance(output_item.summary, list): for summary_item in output_item.summary: - if hasattr(summary_item, "text"): + if getattr(summary_item, "text", None) is not None: summary_item.text = REDACTED_BY_LITELLM if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"): @@ -142,17 +144,17 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str): if not isinstance(output_item, dict): continue - if "text" in output_item: + if output_item.get("text") is not None: output_item["text"] = redacted_str if isinstance(output_item.get("content"), list): for content_item in output_item["content"]: - if isinstance(content_item, dict) and "text" in content_item: + if isinstance(content_item, dict) and content_item.get("text") is not None: content_item["text"] = redacted_str if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list): for summary_item in output_item["summary"]: - if isinstance(summary_item, dict) and "text" in summary_item: + if isinstance(summary_item, dict) and summary_item.get("text") is not None: summary_item["text"] = redacted_str if output_item.get("type") == "function_call" and "arguments" in output_item: @@ -189,40 +191,42 @@ def _redact_standard_logging_object(model_call_details: dict): standard_logging_object["response"] = {"text": redacted_str} -def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None: +def _redact_tool_calls_dict(message: dict) -> None: """Redact tool call / function_call arguments in a dict-form message or delta.""" tool_calls: Final = message.get("tool_calls") if isinstance(tool_calls, list): for tool_call in tool_calls: if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict): - tool_call["function"]["arguments"] = redacted_str + tool_call["function"]["arguments"] = REDACTED_BY_LITELLM function_call: Final = message.get("function_call") if isinstance(function_call, dict) and "arguments" in function_call: - function_call["arguments"] = redacted_str + function_call["arguments"] = REDACTED_BY_LITELLM def _redact_model_response_dict_choices(choices, redacted_str: str): for choice in choices: if isinstance(choice, dict): if "message" in choice and isinstance(choice["message"], dict): - choice["message"]["content"] = redacted_str - if "reasoning_content" in choice["message"]: + if choice["message"].get("content") is not None: + choice["message"]["content"] = redacted_str + if choice["message"].get("reasoning_content") is not None: choice["message"]["reasoning_content"] = redacted_str if "thinking_blocks" in choice["message"]: choice["message"]["thinking_blocks"] = None if "audio" in choice["message"]: choice["message"]["audio"] = None - _redact_tool_calls_dict(choice["message"], redacted_str) + _redact_tool_calls_dict(choice["message"]) elif "delta" in choice and isinstance(choice["delta"], dict): - choice["delta"]["content"] = redacted_str - if "reasoning_content" in choice["delta"]: + if choice["delta"].get("content") is not None: + choice["delta"]["content"] = redacted_str + if choice["delta"].get("reasoning_content") is not None: choice["delta"]["reasoning_content"] = redacted_str if "thinking_blocks" in choice["delta"]: choice["delta"]["thinking_blocks"] = None if "audio" in choice["delta"]: choice["delta"]["audio"] = None - _redact_tool_calls_dict(choice["delta"], redacted_str) + _redact_tool_calls_dict(choice["delta"]) else: _redact_choice_content(choice) @@ -263,7 +267,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse)) or (isinstance(result, dict) and ("choices" in result or "output" in result)) ): - return {"text": "redacted-by-litellm"} + return {"text": REDACTED_BY_LITELLM} _result: Final = copy.deepcopy(result) if isinstance(_result, litellm.ModelResponse): diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 1566bb1bdd7..7798dfeb6bd 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.proxy._types import SpendLogsPayload from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler from litellm.responses.utils import ResponsesAPIRequestUtils @@ -29,6 +30,17 @@ COLD_STORAGE_HANDLER: Final = ColdStorageHandler() ######################################################## +def _normalize_redacted_tool_call_arguments(message: Message) -> None: + """Redaction stores the bare sentinel (invalid JSON) in tool-call arguments; + normalize replayed history to "{}" so provider converters can parse it.""" + for tool_call in message.tool_calls or []: + if (function := getattr(tool_call, "function", None)) is not None and function.arguments == REDACTED_BY_LITELLM: + function.arguments = REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER + function_call: Final = message.function_call + if function_call is not None and function_call.arguments == REDACTED_BY_LITELLM: + function_call.arguments = REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER + + class ResponsesSessionHandler: @staticmethod async def get_chat_completion_message_history_for_previous_response_id( @@ -143,7 +155,8 @@ class ResponsesSessionHandler: model_response: Final = ModelResponse(**_response_output) for choice in model_response.choices: if hasattr(choice, "message"): - chat_completion_message_history.append(getattr(choice, "message")) + _normalize_redacted_tool_call_arguments(message := getattr(choice, "message")) + chat_completion_message_history.append(message) return chat_completion_message_history @staticmethod diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b8d7b726a28..7beb62d2ea3 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -32,6 +32,7 @@ from typing_extensions import TypedDict from litellm._logging import verbose_logger from litellm.caching import InMemoryCache +from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -1551,6 +1552,9 @@ class LiteLLMCompletionResponsesConfig: # store their payload in "input" (raw string) rather than # "arguments" (JSON string), so normalize to arguments here. raw_arguments = function_call.get("arguments") + if raw_arguments == REDACTED_BY_LITELLM: + # redaction stores the bare sentinel (invalid JSON) in arguments + raw_arguments = REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER if not raw_arguments and function_call.get("type") == "custom_tool_call": raw_input: Final = function_call.get("input") or "" raw_arguments = json.dumps({"content": raw_input}) if raw_input else "" diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 417921c166b..89607494920 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -933,6 +933,52 @@ class TestLangfuseOtelResponsesAPI: assert output_data[0]["arguments"]["location"] == "San Francisco" assert output_data[0]["arguments"]["unit"] == "celsius" + def test_responses_api_function_call_with_redacted_arguments(self): + """Sentinel arguments (invalid JSON) must not kill the whole observation output.""" + from openai.types.responses import ResponseFunctionToolCall + + from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes + + response_obj = ResponsesAPIResponse( + id="response-redacted", + created_at=1625247700, + output=[ + ResponseFunctionToolCall( + id="fc-redacted", + type="function_call", + name="get_weather", + call_id="call-redacted", + arguments="redacted-by-litellm", + status="completed", + ) + ], + ) + + kwargs = { + "call_type": "responses", + "messages": [{"role": "user", "content": "What's the weather?"}], + "model": "gpt-4o", + "optional_params": {}, + } + + mock_span = MagicMock() + + with patch( # test-quality-ok: the span attribute sink is the observable boundary; sibling tests in this class stub the same seam + "litellm.integrations.arize._utils.safe_set_attribute" + ) as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, kwargs, response_obj) + + output_calls = [ + call + for call in mock_safe_set_attribute.call_args_list + if call.args[1] == LangfuseSpanAttributes.OBSERVATION_OUTPUT.value + ] + + assert len(output_calls) > 0, "observation.output should still be set" + output_data = json.loads(output_calls[0].args[2]) + assert output_data[0]["name"] == "get_weather" + assert output_data[0]["arguments"] == {} + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 8fa6d44dd8a..3be0bae4120 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -350,7 +350,7 @@ class TestPerformRedaction: redacted = perform_redaction({}, result) message = redacted["choices"][0]["message"] - assert message["content"] == "redacted-by-litellm" + assert message["content"] is None tool_call = message["tool_calls"][0] assert tool_call["function"]["arguments"] == "redacted-by-litellm" assert tool_call["function"]["name"] == "get_weather" @@ -491,6 +491,76 @@ class TestPerformRedaction: assert redacted["output"][0]["arguments"] == "redacted-by-litellm" assert redacted["output"][0]["name"] == "get_weather" + def test_redacts_every_tool_call_in_multi_element_list(self): + result = litellm.ModelResponse( + id="resp-multi", + choices=[ + litellm.Choices( + message=litellm.Message( + content=None, + role="assistant", + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "a"}'}, + }, + { + "id": "call_2", + "type": "function", + "function": {"name": "get_time", "arguments": '{"tz": "b"}'}, + }, + ], + ) + ) + ], + model="gpt-4o", + ) + + redacted = perform_redaction({}, result) + + tool_calls = redacted.choices[0].message.tool_calls + assert tool_calls[0].function.arguments == "redacted-by-litellm" + assert tool_calls[1].function.arguments == "redacted-by-litellm" + + def test_preserves_none_content_on_tool_call_only_message(self): + result = litellm.ModelResponse( + id="resp-none", + choices=[ + litellm.Choices( + message=litellm.Message( + content=None, + role="assistant", + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "a"}'}, + } + ], + ) + ) + ], + model="gpt-4o", + ) + + redacted = perform_redaction({}, result) + + assert redacted.choices[0].message.content is None + + def test_redacts_responses_api_function_call_arguments_object(self): + output_item = SimpleNamespace( + type="function_call", + name="get_weather", + arguments='{"city": "sensitive-city"}', + call_id="call_1", + ) + + _redact_responses_api_output([output_item]) + + assert output_item.arguments == "redacted-by-litellm" + assert output_item.name == "get_weather" + def test_redacts_response_output_objects_with_top_level_text(self): output_items = [ SimpleNamespace(text="top-level output"), @@ -502,6 +572,29 @@ class TestPerformRedaction: assert output_items[0].text == "redacted-by-litellm" assert output_items[1] == "non-dict output item" + def test_preserves_none_text_in_responses_output(self): + from litellm.litellm_core_utils.redact_messages import _redact_responses_api_output_dict + + none_item = SimpleNamespace(type="output_text", text=None, content=[SimpleNamespace(text=None)]) + real_item = SimpleNamespace(type="output_text", text="real answer", content=[SimpleNamespace(text="real part")]) + + _redact_responses_api_output([none_item, real_item]) + + assert none_item.text is None + assert none_item.content[0].text is None + assert real_item.text == "redacted-by-litellm" + assert real_item.content[0].text == "redacted-by-litellm" + + none_dict = {"type": "output_text", "text": None, "content": [{"text": None}]} + real_dict = {"type": "output_text", "text": "real answer", "content": [{"text": "real part"}]} + + _redact_responses_api_output_dict([none_dict, real_dict], "redacted-by-litellm") + + assert none_dict["text"] is None + assert none_dict["content"][0]["text"] is None + assert real_dict["text"] == "redacted-by-litellm" + assert real_dict["content"][0]["text"] == "redacted-by-litellm" + def test_skips_non_dict_response_output_items(self): result = { "output": [ diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2273f23b1cc..b96d2eb5322 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -953,6 +953,19 @@ class TestFunctionCallTransformation: assert function.get("name") == "get_weather" assert function.get("arguments") == '{"location": "São Paulo, Brazil"}' + def test_function_call_transformation_normalizes_redacted_arguments(self): + """Redacted rows hold the bare sentinel in arguments, which is invalid JSON.""" + result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call={ + "type": "function_call", + "name": "get_weather", + "arguments": "redacted-by-litellm", + "call_id": "call_123", + } + ) + + assert result[0]["tool_calls"][0]["function"]["arguments"] == "{}" + def test_complete_input_transformation_with_function_calls(self): """Test the complete transformation with the exact input from the issue""" test_input = [ diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py index df477f6d01e..901fa8f57ff 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py @@ -9,8 +9,10 @@ import litellm from litellm.responses.litellm_completion_transformation import session_handler from litellm.responses.litellm_completion_transformation.session_handler import ( ResponsesSessionHandler, + _normalize_redacted_tool_call_arguments, ) from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.utils import Message @pytest.mark.asyncio @@ -638,3 +640,81 @@ async def test_session_lookup_does_not_retry_when_spend_logs_are_disabled( assert spend_logs == [] assert fake_prisma_client.db.calls == [("chatcmpl-does-not-exist",)] + + +def test_normalize_redacted_arguments_skips_custom_tool_calls(): + """Custom tool calls have no .function; the normalizer must skip them, not crash (session replay path).""" + message = Message( + content=None, + tool_calls=[ + {"id": "call_c", "type": "custom", "custom": {"name": "run_code", "input": "print(1)"}}, + {"id": "call_f", "type": "function", "function": {"name": "get_weather", "arguments": "redacted-by-litellm"}}, + ], + ) + + _normalize_redacted_tool_call_arguments(message) + + assert message.tool_calls[0].custom.input == "print(1)" + assert message.tool_calls[1].function.arguments == "{}" + + +@pytest.mark.asyncio +async def test_message_history_normalizes_redacted_tool_call_arguments(): + """Sessions stored with turn_off_message_logging hold the bare sentinel + in tool-call arguments; replay must normalize it to valid JSON.""" + mock_spend_logs = [ + { + "request_id": "chatcmpl-redacted-1", + "call_type": "aresponses", + "session_id": "sess-redacted", + "proxy_server_request": { + "input": "what is the weather in sf", + "model": "gpt-4o", + }, + "response": { + "id": "chatcmpl-redacted-1", + "model": "gpt-4o", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "redacted-by-litellm", + }, + } + ], + "function_call": None, + }, + "finish_reason": "tool_calls", + } + ], + "created": 1748575031, + "usage": {"total_tokens": 10, "prompt_tokens": 5, "completion_tokens": 5}, + }, + "status": "success", + } + ] + + with patch.object( # test-quality-ok: the handler has no DI seam for the spend-log fetch; every test in this file stubs this same boundary + ResponsesSessionHandler, + "get_all_spend_logs_for_previous_response_id", + new_callable=AsyncMock, + ) as mock_get_spend_logs: + mock_get_spend_logs.return_value = mock_spend_logs + + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + "chatcmpl-redacted-1" + ) + + assistant_message = result["messages"][-1] + tool_call = assistant_message.tool_calls[0] + assert tool_call.function.arguments == "{}" + assert json.loads(tool_call.function.arguments) == {}