diff --git a/litellm/utils.py b/litellm/utils.py index 7b02367e0bf..3acd9e8bca5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1037,85 +1037,106 @@ def is_gemini_tts_model(model: str) -> bool: return "gemini" in model_name and "tts" in model_name -def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str: - """ - Remove thought signature from a tool call ID. - """ - if separator in tool_call_id: - return tool_call_id.split(separator, 1)[0] - return tool_call_id +_THOUGHT_SIGNATURE_CALL_TYPES: Final = frozenset( + {CallTypes.completion.value, CallTypes.acompletion.value, CallTypes.anthropic_messages.value} +) +_MESSAGE_FIELDS: Final = TypeAdapter(Mapping[str, object]) -def _process_assistant_message_tool_calls(msg_copy: dict, thought_signature_separator: str) -> dict: - """ - Process assistant message to remove thought signatures from tool call IDs. - """ - role: Final = msg_copy.get("role") - tool_calls: Final = msg_copy.get("tool_calls") - - if role == "assistant" and isinstance(tool_calls, list): - new_tool_calls: Final = [] - for tc in tool_calls: - # Handle both dict and Pydantic model tool calls - if hasattr(tc, "model_dump"): - # It's a Pydantic model, convert to dict - tc_dict = tc.model_dump() - elif isinstance(tc, dict): - tc_dict = tc.copy() - else: - new_tool_calls.append(tc) - continue - - # Remove thought signature from ID if present - if isinstance(tc_dict.get("id"), str): - if thought_signature_separator in tc_dict["id"]: - tc_dict["id"] = _remove_thought_signature_from_id(tc_dict["id"], thought_signature_separator) - - new_tool_calls.append(tc_dict) - msg_copy["tool_calls"] = new_tool_calls - - return msg_copy +class _ToolCallIdRef(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + id: str | None = None -def _process_tool_message_id(msg_copy: dict, thought_signature_separator: str) -> dict: - """ - Process tool message to remove thought signature from tool_call_id. - """ - if msg_copy.get("role") == "tool" and isinstance(msg_copy.get("tool_call_id"), str): - if thought_signature_separator in msg_copy["tool_call_id"]: - msg_copy["tool_call_id"] = _remove_thought_signature_from_id( - msg_copy["tool_call_id"], thought_signature_separator - ) - - return msg_copy +class _MessageToolCallIds(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + tool_call_id: str | None = None + tool_calls: tuple[_ToolCallIdRef, ...] | None = None -def _remove_thought_signatures_from_messages(messages: list, thought_signature_separator: str) -> list: - """ - Remove thought signatures from tool call IDs in all messages. - """ - processed_messages: Final = [] +def _tool_call_ids(message: object) -> tuple[str | None, ...]: + try: + refs: Final = _MessageToolCallIds.model_validate(message) + except ValidationError: + return () + return (refs.tool_call_id, *(call.id for call in refs.tool_calls or ())) - for msg in messages: - # Handle Pydantic models (convert to dict) - if hasattr(msg, "model_dump"): - msg_dict = msg.model_dump() - elif isinstance(msg, dict): - msg_dict = msg.copy() - else: - # Unknown type, keep as is - processed_messages.append(msg) - continue - # Process assistant messages with tool_calls - msg_dict = _process_assistant_message_tool_calls(msg_dict, thought_signature_separator) +def _fields(value: object) -> Mapping[str, object] | None: + try: + return _MESSAGE_FIELDS.validate_python(value.model_dump() if isinstance(value, BaseModel) else value) + except ValidationError: + return None - # Process tool messages with tool_call_id - msg_dict = _process_tool_message_id(msg_dict, thought_signature_separator) - processed_messages.append(msg_dict) +def _unsigned(value: object, separator: str) -> object: + return value.split(separator, 1)[0] if isinstance(value, str) else value - return processed_messages + +def _unsigned_tool_call(call: object, separator: str) -> object: + fields: Final = _fields(call) + if fields is None: + return call + return {key: _unsigned(value, separator) if key == "id" else value for key, value in fields.items()} + + +def _unsigned_tool_calls(value: object, separator: str) -> object: + try: + calls: Final = _TOOL_LIST.validate_python(value, strict=True) + except ValidationError: + return value + return [_unsigned_tool_call(call, separator) for call in calls] + + +def _unsigned_message_field(role: object, key: str, value: object, separator: str) -> object: + if role == "assistant" and key == "tool_calls": + return _unsigned_tool_calls(value, separator) + if role == "tool" and key == "tool_call_id": + return _unsigned(value, separator) + return value + + +def _unsigned_message(message: object, separator: str) -> object: + fields: Final = _fields(message) + if fields is None: + return message + role: Final = fields.get("role") + return {key: _unsigned_message_field(role, key, value, separator) for key, value in fields.items()} + + +def _target_provider(model: str, declared: object) -> str | None: + if isinstance(declared, str) and declared: + return declared + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + + try: + return resolve_model_and_provider_for_metadata(model=model)[1] + except Exception: + return None + + +def messages_without_foreign_thought_signatures( + call_type: str, model: object, messages: object, custom_llm_provider: object +) -> Sequence[object] | None: + """Messages with Gemini thought signatures cut from tool call ids, or None when the call's target keeps them. + + Must run per LLM call with the deployment's model, never on the proxy's alias before routing.""" + from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR + + if call_type not in _THOUGHT_SIGNATURE_CALL_TYPES or not isinstance(model, str): + return None + try: + message_list: Final = _TOOL_LIST.validate_python(messages, strict=True) + except ValidationError: + return None + signed: Final = any( + tool_call_id is not None and THOUGHT_SIGNATURE_SEPARATOR in tool_call_id + for tool_call_id in itertools.chain.from_iterable(map(_tool_call_ids, message_list)) + ) + if not signed or _is_gemini_model(model, _target_provider(model, custom_llm_provider)): + return None + verbose_logger.debug("Removing thought signatures from tool call IDs for non-Gemini model %s", model) + return [_unsigned_message(message, THOUGHT_SIGNATURE_SEPARATOR) for message in message_list] def _restore_correlation_context_if_supported(logging_obj: object) -> None: @@ -1492,55 +1513,6 @@ def function_setup( input=buffer.getvalue(), model=model, ) - - ### REMOVE THOUGHT SIGNATURES FROM TOOL CALL IDS FOR NON-GEMINI MODELS ### - # Gemini models embed thought signatures in tool call IDs. When sending - # messages with tool calls to non-Gemini providers, we need to remove these - # signatures to ensure compatibility. - if isinstance(messages, list) and len(messages) > 0: - try: - from litellm.litellm_core_utils.get_llm_provider_logic import ( - resolve_model_and_provider_for_metadata, - ) - from litellm.litellm_core_utils.prompt_templates.factory import ( - THOUGHT_SIGNATURE_SEPARATOR, - ) - - # Get custom_llm_provider to determine target provider - custom_llm_provider = kwargs.get("custom_llm_provider") - - # If custom_llm_provider not in kwargs, try to determine it from the model - metadata_model: Final = _OPAQUE_VALUE.validate_python(model) - if not custom_llm_provider and isinstance(metadata_model, str) and metadata_model: - try: - _, custom_llm_provider = resolve_model_and_provider_for_metadata( - model=metadata_model, - custom_llm_provider=custom_llm_provider, - ) - except Exception: - # If we can't determine the provider, skip this processing - pass - - # Only process if target is NOT a Gemini model - if not _is_gemini_model(model, custom_llm_provider): - verbose_logger.debug("Removing thought signatures from tool call IDs for non-Gemini model") - - # Process messages to remove thought signatures - processed_messages: Final = _remove_thought_signatures_from_messages( - messages, THOUGHT_SIGNATURE_SEPARATOR - ) - - # Update messages in kwargs or args - if "messages" in kwargs: - kwargs["messages"] = processed_messages - elif len(args) > 1: - args_list: Final = list(args) - args_list[1] = processed_messages - args = tuple(args_list) - - except Exception as e: - # Log the error but don't fail the request - verbose_logger.warning("Error removing thought signatures from tool call IDs: %s", e) elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value: messages = args[1] if len(args) > 1 else kwargs.get("input", None) elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value: @@ -2300,6 +2272,14 @@ def client(original_function): # Type assertion: logging_obj is guaranteed to be non-None after function_setup assert logging_obj is not None, "logging_obj should not be None after function_setup" + signature_free_messages: Final = messages_without_foreign_thought_signatures( + call_type, + _OPAQUE_VALUE.validate_python(model), + _OPAQUE_VALUE.validate_python(kwargs.get("messages")), + _OPAQUE_VALUE.validate_python(kwargs.get("custom_llm_provider")), + ) + if signature_free_messages is not None: + kwargs["messages"] = signature_free_messages ## LOAD CREDENTIALS load_credentials_from_list(kwargs, call_type) @@ -2569,6 +2549,14 @@ def client(original_function): # Type assertion: logging_obj is guaranteed to be non-None after function_setup assert logging_obj is not None, "logging_obj should not be None after function_setup" + signature_free_messages: Final = messages_without_foreign_thought_signatures( + call_type, + _OPAQUE_VALUE.validate_python(model), + _OPAQUE_VALUE.validate_python(kwargs.get("messages")), + _OPAQUE_VALUE.validate_python(kwargs.get("custom_llm_provider")), + ) + if signature_free_messages is not None: + kwargs["messages"] = signature_free_messages if not _is_litellm_internal_call: bind_budget_reservation_to_callbacks(logging_obj.litellm_params) diff --git a/tests/integration/providers/test_vertex_gemini_function_call_wire.py b/tests/integration/providers/test_vertex_gemini_function_call_wire.py index fc806fdfb85..ce1ee7897c0 100644 --- a/tests/integration/providers/test_vertex_gemini_function_call_wire.py +++ b/tests/integration/providers/test_vertex_gemini_function_call_wire.py @@ -13,6 +13,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from integration._support.client import Gateway, Scenario from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.litellm_core_utils.prompt_templates.factory import get_dummy_thought_signature @@ -238,6 +239,152 @@ def test_vertex_gemini_kwargs_extra_param_reaches_generation_config(gateway: Gat assert response.json()["choices"][0]["message"]["content"] == "done" +_REPLAYED_SIGNATURE: Final = "cmVwbGF5ZWQtc2lnbmF0dXJl" +_SIGNED_CALL_ID: Final = f"call_7c1e__thought__{_REPLAYED_SIGNATURE}" +_TOOL_TURN_MESSAGES: Final = [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": _SIGNED_CALL_ID, + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps(_ARGS)}, + } + ], + }, + {"role": "tool", "tool_call_id": _SIGNED_CALL_ID, "content": "sunny"}, +] + + +def _tool_turn_call(gateway: Gateway, model: str, **extra: JsonValue) -> None: + response: Final = gateway.client.post( + "/v1/chat/completions", + json={ + "model": model, + "tools": [{"type": "function", "function": function} for function in _FUNCTIONS], + "messages": _TOOL_TURN_MESSAGES, + **extra, + }, + headers={"Authorization": f"Bearer {gateway.key}"}, + timeout=30, + ) + assert response.status_code == 200, response.text + + +class _GeminiPart(TypedDict): + function_call: NotRequired[ReadOnly[JsonValue]] + thoughtSignature: NotRequired[ReadOnly[str]] + + +class _GeminiContent(TypedDict): + parts: ReadOnly[tuple[_GeminiPart, ...]] + + +class _GeminiRequest(TypedDict): + contents: ReadOnly[tuple[_GeminiContent, ...]] + + +class _ToolCall(TypedDict): + id: ReadOnly[str] + + +class _ChatMessage(TypedDict): + tool_calls: NotRequired[ReadOnly[tuple[_ToolCall, ...] | None]] + tool_call_id: NotRequired[ReadOnly[str]] + + +class _ChatRequest(TypedDict): + messages: ReadOnly[tuple[_ChatMessage, ...]] + + +_GEMINI_REQUEST: Final = TypeAdapter(_GeminiRequest) +_CHAT_REQUEST: Final = TypeAdapter(_ChatRequest) + + +def _function_call_signatures(request: Request) -> tuple[str | None, ...]: + contents: Final = _GEMINI_REQUEST.validate_json(request.body)["contents"] + parts: Final = itertools.chain.from_iterable(content["parts"] for content in contents) + return tuple(part.get("thoughtSignature") for part in parts if "function_call" in part) + + +def _sent_tool_call_ids(request: Request) -> tuple[str, ...]: + messages: Final = _CHAT_REQUEST.validate_json(request.body)["messages"] + calls: Final = itertools.chain.from_iterable(message.get("tool_calls") or () for message in messages) + return ( + *(call["id"] for call in calls), + *(message["tool_call_id"] for message in messages if "tool_call_id" in message), + ) + + +def test_vertex_gemini_alias_without_gemini_in_name_replays_tool_call_thought_signature(gateway: Gateway) -> None: + with ( + wire_server(lambda _: Reply(body=json.dumps(_candidate(with_signature=False)).encode())) as wire, + gateway.scenario() as scenario, + ): + model: Final = _model(gateway, scenario, wire.url) + assert "gemini" not in model + _tool_turn_call(gateway, model) + (request,) = wire.drain() + + assert request.target == f"{_MODEL_PATH}:generateContent" + assert _function_call_signatures(request) == (_REPLAYED_SIGNATURE,) + + +def test_openai_target_receives_tool_call_ids_without_thought_signature(gateway: Gateway) -> None: + completion: Final = { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "created": 0, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5}, + } + with wire_server(lambda _: Reply(body=json.dumps(completion).encode())) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=f"{wire.url}/v1") + _tool_turn_call(gateway, model) + (request,) = wire.drain() + + assert request.target == "/v1/chat/completions" + assert _sent_tool_call_ids(request) == ("call_7c1e", "call_7c1e") + + +def test_vertex_gemini_alias_without_gemini_in_name_replays_tool_call_thought_signature_streaming( + gateway: Gateway, +) -> None: + payload: Final = json.dumps(_candidate(with_signature=False)) + with ( + wire_server( + lambda _: Reply(content_type="text/event-stream", chunks=(f"data: {payload}\n\n".encode(),)) + ) as wire, + gateway.scenario() as scenario, + ): + model: Final = _model(gateway, scenario, wire.url) + _tool_turn_call(gateway, model, stream=True) + (request,) = wire.drain() + + assert request.target == f"{_MODEL_PATH}:streamGenerateContent?alt=sse" + assert _function_call_signatures(request) == (_REPLAYED_SIGNATURE,) + + +def test_fallback_from_openai_to_vertex_gemini_alias_strips_then_replays_thought_signature(gateway: Gateway) -> None: + with ( + wire_server(lambda _: Reply(status=500, body=b'{"error": {"message": "scripted outage"}}')) as openai_wire, + wire_server(lambda _: Reply(body=json.dumps(_candidate(with_signature=False)).encode())) as vertex_wire, + gateway.scenario() as scenario, + ): + primary: Final = scenario.model(api_base=f"{openai_wire.url}/v1") + fallback: Final = _model(gateway, scenario, vertex_wire.url) + _tool_turn_call(gateway, primary, fallbacks=[fallback], num_retries=0) + openai_requests: Final = openai_wire.drain() + (vertex_request,) = vertex_wire.drain() + + assert openai_requests, "the primary deployment was never tried" + assert {_sent_tool_call_ids(request) for request in openai_requests} == {("call_7c1e", "call_7c1e")} + assert _function_call_signatures(vertex_request) == (_REPLAYED_SIGNATURE,) + + Provider: TypeAlias = Literal["gemini", "vertex_ai"] _GEMINI_3_BACKEND: Final = "gemini-3.8-flash" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 56dcf8af9fa..ee7d8e8c4c5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -85,6 +85,7 @@ from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelRe from litellm.utils import ( Rules, function_setup, + messages_without_foreign_thought_signatures, validate_and_fix_openai_messages, ) from litellm.litellm_core_utils.model_cost_table import publish @@ -321,7 +322,11 @@ def test_ollama_pt_consecutive_user_messages(): def _ollama_tool_turn(*results: object) -> list[dict]: - call: Final = {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + call: Final = { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } return [ {"role": "user", "content": "Weather in Paris?"}, {"role": "assistant", "content": None, "tool_calls": [call]}, @@ -1377,7 +1382,6 @@ def test_anthropic_messages_pt_sends_a_signed_block_once_when_thinking_blocks_ho assert result[1] == {"role": "assistant", "content": [signed, redacted, {"type": "text", "text": "hello"}]} - def test_anthropic_messages_pt_sends_a_block_once_when_two_assistant_messages_carry_it(): signed: Final = {"type": "thinking", "thinking": "Let me look this up.", "signature": "sig-abc"} redacted: Final = {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA} @@ -1397,6 +1401,7 @@ def test_anthropic_messages_pt_sends_a_block_once_when_two_assistant_messages_ca ], } + @REASONING_PLACEMENTS @pytest.mark.parametrize("drop_params", [False, True]) def test_anthropic_messages_pt_drops_cache_control_from_thinking_blocks_only_under_drop_params( @@ -2185,7 +2190,11 @@ def test_convert_to_gemini_tool_call_invoke_drops_unusable_arguments_under_drop_ function: Final = {"name": "get_weather", "arguments": arguments} message: Final = ( - {"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function", "function": function}]} + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": function}], + } if shape == "tool_calls" else {"role": "assistant", "content": None, "function_call": function} ) @@ -2429,10 +2438,11 @@ def test_gemini_tool_media_keeps_vertex_reference_when_inline_data_is_absent( ] - def test_gemini_tool_media_fetch_failure_preserves_tool_text() -> None: message: Final = ChatCompletionToolMessage( - role="tool", name="lookup", content=[ + role="tool", + name="lookup", + content=[ {"type": "text", "text": '{"ok":true}'}, {"type": "image_url", "image_url": {"url": "https://8.8.8.8/unavailable.png"}}, ], @@ -2442,6 +2452,7 @@ def test_gemini_tool_media_fetch_failure_preserves_tool_text() -> None: result: Final = convert_to_gemini_tool_call_result(message, None, custom_llm_provider="gemini") assert result == {"function_response": {"name": "lookup", "response": {"ok": True}}} + def test_convert_gemini_tool_call_result_with_anthropic_image_block(): """ Test that Anthropic-native image blocks in tool_result list content are @@ -6749,181 +6760,74 @@ def test_empty_content(): ) -@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") -def test_thought_signature_removal_for_non_gemini(): - """ - Test that thought signatures are removed from tool call IDs when sending to non-Gemini models - """ - rules_obj = Rules() - - # Create messages with thought signatures (as would come from Gemini) - messages = [ +def _signed_tool_turn(*call_ids: str) -> Sequence[Mapping[str, object]]: + return [ {"role": "user", "content": "What's the weather?"}, { "role": "assistant", "tool_calls": [ { - "id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"location": "SF"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1", - "content": "Sunny, 72°F", - }, - ] - - # Call function_setup with OpenAI model (non-Gemini) - logging_obj, kwargs = function_setup( - original_function="acompletion", - rules_obj=rules_obj, - start_time=datetime.now(), - model="gpt-4", - messages=messages, - litellm_call_id=str(uuid.uuid4()), - custom_llm_provider="openai", - ) - - # Verify thought signatures were removed - processed_messages = kwargs["messages"] - assert processed_messages[1]["tool_calls"][0]["id"] == "call_123" - assert processed_messages[2]["tool_call_id"] == "call_123" - assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[1]["tool_calls"][0]["id"] - assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[2]["tool_call_id"] - - -@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") -def test_thought_signature_preserved_for_gemini(): - """ - Test that thought signatures are preserved when sending to Gemini models - """ - rules_obj = Rules() - - # Create messages with thought signatures - messages = [ - {"role": "user", "content": "What's the weather?"}, - { - "role": "assistant", - "tool_calls": [ - { - "id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"location": "NYC"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2", - "content": "Rainy, 65°F", - }, - ] - - # Call function_setup with Gemini model - logging_obj, kwargs = function_setup( - original_function="acompletion", - rules_obj=rules_obj, - start_time=datetime.now(), - model="gemini-1.5-pro", - messages=messages, - litellm_call_id=str(uuid.uuid4()), - custom_llm_provider="vertex_ai", - ) - - # Verify thought signatures were preserved (messages should be unchanged) - processed_messages = kwargs["messages"] - assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[1]["tool_calls"][0]["id"] - assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[2]["tool_call_id"] - - -@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") -def test_thought_signature_removal_with_multiple_tool_calls(): - """ - Test that thought signatures are removed from multiple tool calls - """ - rules_obj = Rules() - - messages = [ - {"role": "user", "content": "Get weather and time"}, - { - "role": "assistant", - "tool_calls": [ - { - "id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1", + "id": f"{call_id}{THOUGHT_SIGNATURE_SEPARATOR}sig{index}", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}, - }, - { - "id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2", - "type": "function", - "function": {"name": "get_time", "arguments": "{}"}, - }, + } + for index, call_id in enumerate(call_ids) ], }, - { - "role": "tool", - "tool_call_id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1", - "content": "Sunny", - }, - { - "role": "tool", - "tool_call_id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2", - "content": "3:00 PM", - }, + *( + {"role": "tool", "tool_call_id": f"{call_id}{THOUGHT_SIGNATURE_SEPARATOR}sig{index}", "content": "Sunny"} + for index, call_id in enumerate(call_ids) + ), ] - logging_obj, kwargs = function_setup( - original_function="acompletion", - rules_obj=rules_obj, - start_time=datetime.now(), - model="claude-3-opus", - messages=messages, - litellm_call_id=str(uuid.uuid4()), - custom_llm_provider="anthropic", + +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [("gpt-4", "openai"), ("claude-3-opus", "anthropic"), ("openai/gpt-4", None)], +) +def test_thought_signatures_are_cut_from_every_tool_call_id_for_a_non_gemini_target(model, custom_llm_provider): + stripped = messages_without_foreign_thought_signatures( + "acompletion", model, _signed_tool_turn("call_1", "call_2"), custom_llm_provider ) - processed_messages = kwargs["messages"] + assert stripped is not None + assert [call["id"] for call in stripped[1]["tool_calls"]] == ["call_1", "call_2"] + assert [message["tool_call_id"] for message in stripped[2:]] == ["call_1", "call_2"] - # Check all tool call IDs are cleaned - assert processed_messages[1]["tool_calls"][0]["id"] == "call_1" - assert processed_messages[1]["tool_calls"][1]["id"] == "call_2" - assert processed_messages[2]["tool_call_id"] == "call_1" - assert processed_messages[3]["tool_call_id"] == "call_2" + +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [("gemini-1.5-pro", "vertex_ai"), ("vertex_ai/gemini-3.8-flash", None), ("gemini/gemini-3.8-flash", None)], +) +def test_thought_signatures_are_kept_for_a_gemini_target(model, custom_llm_provider): + assert ( + messages_without_foreign_thought_signatures( + "acompletion", model, _signed_tool_turn("call_1"), custom_llm_provider + ) + is None + ) + + +def test_messages_without_signed_tool_call_ids_are_left_alone_for_a_non_gemini_target(): + messages = [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there!"}] + + assert messages_without_foreign_thought_signatures("acompletion", "gpt-4", messages, "openai") is None @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") -def test_messages_without_tool_calls_unchanged(): - """ - Test that messages without tool calls pass through unchanged - """ - rules_obj = Rules() +def test_proxy_request_setup_keeps_signatures_for_a_model_name_without_gemini_in_it(): + messages = _signed_tool_turn("call_1") - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - - logging_obj, kwargs = function_setup( + _, kwargs = function_setup( original_function="acompletion", - rules_obj=rules_obj, + rules_obj=Rules(), start_time=datetime.now(), - model="gpt-4", + model="g38flash", messages=messages, litellm_call_id=str(uuid.uuid4()), - custom_llm_provider="openai", ) - # Messages should be unchanged - assert kwargs["messages"] == messages + assert kwargs["messages"] is messages, "the proxy alias must not decide signature stripping before routing" def test_llama_3_prompt(): @@ -8990,7 +8894,6 @@ def test_anthropic_messages_pt_interleave_more_thinking_than_tool_groups(): ], f"Expected order but got: {types}" - def _paused_web_search_turn( thinking_blocks: list[dict[str, str]] | None, content: str | list[dict[str, str]] = "Step 1 is Tokyo." ) -> dict[str, object]: @@ -9453,7 +9356,11 @@ TEMPLATE_INITIAL = "" TEMPLATE_FINAL = "" PARTIAL_TEMPLATE = {"roles": TEMPLATE_ROLES} -FULL_TEMPLATE = {"roles": TEMPLATE_ROLES, "initial_prompt_value": TEMPLATE_INITIAL, "final_prompt_value": TEMPLATE_FINAL} +FULL_TEMPLATE = { + "roles": TEMPLATE_ROLES, + "initial_prompt_value": TEMPLATE_INITIAL, + "final_prompt_value": TEMPLATE_FINAL, +} ROLELESS_TEMPLATE = {"initial_prompt_value": TEMPLATE_INITIAL, "final_prompt_value": TEMPLATE_FINAL} @@ -9675,7 +9582,9 @@ def test_template_wraps_only_the_roles_it_defines_across_a_conversation(monkeypa pytest.param({"roles": {"user": {**TEMPLATE_ROLES["user"], "priority": 1}}}, id="extra-role-key"), pytest.param({"roles": {**TEMPLATE_ROLES, "tool": {"pre_message": 5}}}, id="unused-role-wrong-type"), pytest.param({"roles": {**TEMPLATE_ROLES, 1: {"pre_message": "x"}}}, id="unused-non-string-role"), - pytest.param({"roles": TEMPLATE_ROLES, "initial_prompt_value": None, "bos_token": 7}, id="null-and-unread-keys"), + pytest.param( + {"roles": TEMPLATE_ROLES, "initial_prompt_value": None, "bos_token": 7}, id="null-and-unread-keys" + ), ], ) def test_template_shapes_main_rendered_keep_rendering(template, monkeypatch):