From b9266bb3b9e69b0943c86cce4d2f8c5425ff50f7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 15:25:57 +0530 Subject: [PATCH 1/9] Fix ensure_alternating_roles for correct order --- .../prompt_templates/common_utils.py | 121 ++++++++++++++---- litellm/main.py | 4 + tests/llm_translation/test_prompt_factory.py | 45 +++++++ 3 files changed, 144 insertions(+), 26 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a5d6bc936bb..8791c769af9 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -269,35 +269,56 @@ def _insert_user_continue_message( 2. Final assistant message 3. Consecutive assistant messages - Only inserts messages between consecutive assistant messages, - ignoring all other role types. + Skips tool messages and assistant messages with tool calls in the + alternation check, matching strict templates like llama.cpp. """ if not messages: return messages + def _counts_for_alternation(message: AllMessageValues) -> bool: + role = message.get("role") + if role == "user": + return True + if role == "assistant": + return not bool(message.get("tool_calls")) + return False + result_messages = messages.copy() # Don't modify the input list continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE - # Handle first message if it's an assistant message - if result_messages[0]["role"] == "assistant": + # Handle first counted message if it's an assistant message + if ( + result_messages[0]["role"] == "assistant" + and _counts_for_alternation(result_messages[0]) + ): result_messages.insert(0, continue_message) - # Handle consecutive assistant messages and final message - i = 1 # Start from second message since we handled first message + # Handle consecutive assistant messages in the counted sequence + i = 1 while i < len(result_messages): curr_message = result_messages[i] - prev_message = result_messages[i - 1] - - # Only check for consecutive assistant messages - # Ignore all other role types - if curr_message["role"] == "assistant" and prev_message["role"] == "assistant": - result_messages.insert(i, continue_message) - i += 2 # Skip over the message we just inserted - else: + if ( + curr_message["role"] == "assistant" + and _counts_for_alternation(curr_message) + ): + j = i - 1 + while j >= 0: + previous_message = result_messages[j] + if _counts_for_alternation(previous_message): + if previous_message["role"] == "assistant": + result_messages.insert(i, continue_message) + i += 2 + break + j -= 1 + if i < len(result_messages): i += 1 # Handle final message - if result_messages[-1]["role"] == "assistant" and ensure_alternating_roles: + if ( + result_messages[-1]["role"] == "assistant" + and _counts_for_alternation(result_messages[-1]) + and ensure_alternating_roles + ): result_messages.append(continue_message) return result_messages @@ -310,6 +331,8 @@ def _insert_assistant_continue_message( ) -> List[AllMessageValues]: """ Add assistant continuation messages between consecutive user messages. + Skips tool messages and assistant messages with tool calls in the + alternation check, matching strict templates like llama.cpp. Args: messages: List of message dictionaries @@ -322,27 +345,73 @@ def _insert_assistant_continue_message( if not ensure_alternating_roles or len(messages) <= 1: return messages + def _counts_for_alternation(message: AllMessageValues) -> bool: + role = message.get("role") + if role == "user": + return True + if role == "assistant": + return not bool(message.get("tool_calls")) + return False + # Create a new list to store modified messages modified_messages: List[AllMessageValues] = [] for i, message in enumerate(messages): modified_messages.append(message) - # Check if we need to insert an assistant message - if ( - i < len(messages) - 1 # Not the last message - and message.get("role") == "user" # Current is user - and messages[i + 1].get("role") == "user" - ): # Next is user - # Insert assistant message - continue_message = ( - assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE - ) - modified_messages.append(continue_message) + if message.get("role") == "user" and _counts_for_alternation(message): + next_counted_index = i + 1 + while next_counted_index < len(messages) and not _counts_for_alternation( + messages[next_counted_index] + ): + next_counted_index += 1 + + if ( + next_counted_index < len(messages) + and messages[next_counted_index].get("role") == "user" + ): + continue_message = ( + assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE + ) + modified_messages.append(continue_message) return modified_messages +def strip_tool_messages_for_alternating_roles( + messages: List[AllMessageValues], +) -> List[AllMessageValues]: + """ + Prepare history for strict user/assistant-only chat templates. + + - Drop tool/function role messages + - Drop assistant tool-dispatch turns with no content + - Keep assistant content turns but remove tool metadata fields + """ + cleaned_messages: List[AllMessageValues] = [] + + for message in messages: + role = message.get("role") + if role in ("tool", "function"): + continue + + if role == "assistant": + assistant_message = message.copy() + assistant_message.pop("tool_calls", None) + assistant_message.pop("function_call", None) + assistant_message.pop("tool_call_id", None) + + if assistant_message.get("content") is None: + continue + + cleaned_messages.append(assistant_message) + continue + + cleaned_messages.append(message) + + return cleaned_messages + + def get_completion_messages( messages: List[AllMessageValues], assistant_continue_message: Optional[ChatCompletionAssistantMessage], diff --git a/litellm/main.py b/litellm/main.py index 81319bc432f..cb5a92caf8d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -166,6 +166,7 @@ from .litellm_core_utils.fallback_utils import ( from .litellm_core_utils.prompt_templates.common_utils import ( add_system_prompt_to_messages, get_completion_messages, + strip_tool_messages_for_alternating_roles, update_messages_with_model_file_ids, ) from .litellm_core_utils.prompt_templates.factory import ( @@ -1298,6 +1299,9 @@ def completion( # type: ignore # noqa: PLR0915 prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) litellm_system_prompt = kwargs.get("litellm_system_prompt", None) ### COPY MESSAGES ### - related issue https://github.com/BerriAI/litellm/discussions/4489 + if ensure_alternating_roles: + messages = strip_tool_messages_for_alternating_roles(messages=messages) + messages = get_completion_messages( messages=messages, ensure_alternating_roles=ensure_alternating_roles or False, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index a6dcabe25ef..2eed9fa2129 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_completion_messages, + strip_tool_messages_for_alternating_roles, ) from litellm.llms.vertex_ai.gemini.transformation import ( _gemini_convert_messages_with_history, @@ -775,6 +776,50 @@ def test_ensure_alternating_roles( assert messages == expected_messages +def test_ensure_alternating_roles_with_tool_calls(): + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_123", "content": "72F, sunny"}, + {"role": "assistant", "content": "It's 72F and sunny in NYC."}, + {"role": "user", "content": "What about tomorrow?"}, + {"role": "user", "content": "And the day after?"}, + {"role": "user", "content": "What about next week?"}, + ] + + messages = strip_tool_messages_for_alternating_roles(messages) + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "What's the weather?"}, + {"role": "assistant", "content": "It's 72F and sunny in NYC."}, + {"role": "user", "content": "What about tomorrow?"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "And the day after?"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "What about next week?"}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From 3cdabff323538df780ce0dc736c22d2f57fd323c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 15:45:09 +0530 Subject: [PATCH 2/9] Fix greptile review --- .../prompt_templates/common_utils.py | 38 ++-------------- litellm/main.py | 4 -- tests/llm_translation/test_prompt_factory.py | 44 +++++++++++++++++-- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 8791c769af9..aa2e07234d5 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -297,6 +297,7 @@ def _insert_user_continue_message( i = 1 while i < len(result_messages): curr_message = result_messages[i] + inserted_continue_message = False if ( curr_message["role"] == "assistant" and _counts_for_alternation(curr_message) @@ -308,9 +309,10 @@ def _insert_user_continue_message( if previous_message["role"] == "assistant": result_messages.insert(i, continue_message) i += 2 + inserted_continue_message = True break j -= 1 - if i < len(result_messages): + if not inserted_continue_message: i += 1 # Handle final message @@ -378,40 +380,6 @@ def _insert_assistant_continue_message( return modified_messages -def strip_tool_messages_for_alternating_roles( - messages: List[AllMessageValues], -) -> List[AllMessageValues]: - """ - Prepare history for strict user/assistant-only chat templates. - - - Drop tool/function role messages - - Drop assistant tool-dispatch turns with no content - - Keep assistant content turns but remove tool metadata fields - """ - cleaned_messages: List[AllMessageValues] = [] - - for message in messages: - role = message.get("role") - if role in ("tool", "function"): - continue - - if role == "assistant": - assistant_message = message.copy() - assistant_message.pop("tool_calls", None) - assistant_message.pop("function_call", None) - assistant_message.pop("tool_call_id", None) - - if assistant_message.get("content") is None: - continue - - cleaned_messages.append(assistant_message) - continue - - cleaned_messages.append(message) - - return cleaned_messages - - def get_completion_messages( messages: List[AllMessageValues], assistant_continue_message: Optional[ChatCompletionAssistantMessage], diff --git a/litellm/main.py b/litellm/main.py index cb5a92caf8d..81319bc432f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -166,7 +166,6 @@ from .litellm_core_utils.fallback_utils import ( from .litellm_core_utils.prompt_templates.common_utils import ( add_system_prompt_to_messages, get_completion_messages, - strip_tool_messages_for_alternating_roles, update_messages_with_model_file_ids, ) from .litellm_core_utils.prompt_templates.factory import ( @@ -1299,9 +1298,6 @@ def completion( # type: ignore # noqa: PLR0915 prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) litellm_system_prompt = kwargs.get("litellm_system_prompt", None) ### COPY MESSAGES ### - related issue https://github.com/BerriAI/litellm/discussions/4489 - if ensure_alternating_roles: - messages = strip_tool_messages_for_alternating_roles(messages=messages) - messages = get_completion_messages( messages=messages, ensure_alternating_roles=ensure_alternating_roles or False, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 2eed9fa2129..12efb47e067 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -25,7 +25,6 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_completion_messages, - strip_tool_messages_for_alternating_roles, ) from litellm.llms.vertex_ai.gemini.transformation import ( _gemini_convert_messages_with_history, @@ -800,8 +799,6 @@ def test_ensure_alternating_roles_with_tool_calls(): {"role": "user", "content": "What about next week?"}, ] - messages = strip_tool_messages_for_alternating_roles(messages) - transformed_messages = get_completion_messages( messages=messages, assistant_continue_message=None, @@ -811,6 +808,21 @@ def test_ensure_alternating_roles_with_tool_calls(): assert transformed_messages == [ {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_123", "content": "72F, sunny"}, {"role": "assistant", "content": "It's 72F and sunny in NYC."}, {"role": "user", "content": "What about tomorrow?"}, {"role": "assistant", "content": "Please continue."}, @@ -820,6 +832,32 @@ def test_ensure_alternating_roles_with_tool_calls(): ] +def test_ensure_alternating_roles_three_consecutive_assistants(): + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "A1"}, + {"role": "assistant", "content": "A2"}, + {"role": "assistant", "content": "A3"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "A1"}, + {"role": "user", "content": "Please continue."}, + {"role": "assistant", "content": "A2"}, + {"role": "user", "content": "Please continue."}, + {"role": "assistant", "content": "A3"}, + {"role": "user", "content": "Please continue."}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From f1421d10825d299fae3f2605c4e48e2cb3655b4f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 15:55:51 +0530 Subject: [PATCH 3/9] fix(prompting): preserve tool chains in alternation insertion Avoid inserting assistant continue messages in the middle of assistant tool_call->tool chains by inserting before the next counted user turn, and add regression coverage for this edge case. Made-with: Cursor --- .../prompt_templates/common_utils.py | 67 +++++++++---------- tests/llm_translation/test_prompt_factory.py | 44 ++++++++++++ 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index aa2e07234d5..739c3119cc0 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -257,6 +257,15 @@ def detect_first_expected_role( return None +def _counts_for_alternation(message: AllMessageValues) -> bool: + role = message.get("role") + if role == "user": + return True + if role == "assistant": + return not bool(message.get("tool_calls")) + return False + + def _insert_user_continue_message( messages: List[AllMessageValues], user_continue_message: Optional[ChatCompletionUserMessage], @@ -275,14 +284,6 @@ def _insert_user_continue_message( if not messages: return messages - def _counts_for_alternation(message: AllMessageValues) -> bool: - role = message.get("role") - if role == "user": - return True - if role == "assistant": - return not bool(message.get("tool_calls")) - return False - result_messages = messages.copy() # Don't modify the input list continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE @@ -346,37 +347,33 @@ def _insert_assistant_continue_message( """ if not ensure_alternating_roles or len(messages) <= 1: return messages - - def _counts_for_alternation(message: AllMessageValues) -> bool: - role = message.get("role") - if role == "user": - return True - if role == "assistant": - return not bool(message.get("tool_calls")) - return False - - # Create a new list to store modified messages - modified_messages: List[AllMessageValues] = [] + continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE + insert_before_indexes = set() for i, message in enumerate(messages): + if message.get("role") != "user": + continue + + next_counted_index = i + 1 + while next_counted_index < len(messages) and not _counts_for_alternation( + messages[next_counted_index] + ): + next_counted_index += 1 + + if ( + next_counted_index < len(messages) + and messages[next_counted_index].get("role") == "user" + ): + # Insert before the next counted user turn. + # This avoids splitting assistant tool-call -> tool chains. + insert_before_indexes.add(next_counted_index) + + modified_messages: List[AllMessageValues] = [] + for idx, message in enumerate(messages): + if idx in insert_before_indexes: + modified_messages.append(continue_message) modified_messages.append(message) - if message.get("role") == "user" and _counts_for_alternation(message): - next_counted_index = i + 1 - while next_counted_index < len(messages) and not _counts_for_alternation( - messages[next_counted_index] - ): - next_counted_index += 1 - - if ( - next_counted_index < len(messages) - and messages[next_counted_index].get("role") == "user" - ): - continue_message = ( - assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE - ) - modified_messages.append(continue_message) - return modified_messages diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 12efb47e067..355c23ae175 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -858,6 +858,50 @@ def test_ensure_alternating_roles_three_consecutive_assistants(): ] +def test_ensure_alternating_roles_does_not_split_tool_call_chain(): + messages = [ + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "user", "content": "Thanks, now do Y"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Thanks, now do Y"}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From 7e4ec1000718f2f35854552eaf0d3aac43f48fa4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:08:15 +0530 Subject: [PATCH 4/9] Update litellm/litellm_core_utils/prompt_templates/common_utils.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/prompt_templates/common_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 739c3119cc0..f5e3ffdb749 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -299,9 +299,7 @@ def _insert_user_continue_message( while i < len(result_messages): curr_message = result_messages[i] inserted_continue_message = False - if ( - curr_message["role"] == "assistant" - and _counts_for_alternation(curr_message) + if _counts_for_alternation(curr_message) and curr_message["role"] == "assistant": ): j = i - 1 while j >= 0: From ae350ed3708b9e90a5a185e9fb5d1073f6b0dc35 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:09:32 +0530 Subject: [PATCH 5/9] Fix greptile comments --- tests/llm_translation/test_prompt_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 355c23ae175..f146e2811a9 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -776,6 +776,7 @@ def test_ensure_alternating_roles( def test_ensure_alternating_roles_with_tool_calls(): + """Fixes Regression in #18685 """ messages = [ {"role": "user", "content": "What's the weather?"}, { From 0941e4036365ad9a57db604d35d98f0356022b54 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:30:42 +0530 Subject: [PATCH 6/9] fix(prompting): address greptile review - fix SyntaxError, restore backward compat, add trailing tool-call test - Remove stray ): on line 303 (P0 SyntaxError) - Restore backward-compatible trailing-assistant behavior (P1) - Add test_ensure_alternating_roles_trailing_tool_call_assistant - Keep role check alongside _counts_for_alternation (P2 is false positive) Made-with: Cursor --- .../prompt_templates/common_utils.py | 10 ++---- tests/llm_translation/test_prompt_factory.py | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index f5e3ffdb749..d7c1cc708b0 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -300,7 +300,6 @@ def _insert_user_continue_message( curr_message = result_messages[i] inserted_continue_message = False if _counts_for_alternation(curr_message) and curr_message["role"] == "assistant": - ): j = i - 1 while j >= 0: previous_message = result_messages[j] @@ -314,12 +313,9 @@ def _insert_user_continue_message( if not inserted_continue_message: i += 1 - # Handle final message - if ( - result_messages[-1]["role"] == "assistant" - and _counts_for_alternation(result_messages[-1]) - and ensure_alternating_roles - ): + # Handle final message — append user_continue after any trailing assistant, + # including ones with tool_calls, to preserve backward compatibility. + if result_messages[-1]["role"] == "assistant" and ensure_alternating_roles: result_messages.append(continue_message) return result_messages diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index f146e2811a9..b02ed3ebea1 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -903,6 +903,40 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ] +def test_ensure_alternating_roles_trailing_tool_call_assistant(): + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + # Backward compat: trailing assistant (even with tool_calls) gets user_continue + # appended, then assistant_continue bridges the user→user gap. + assert transformed_messages[-1] == {"role": "user", "content": "Please continue."} + assert transformed_messages[0] == {"role": "user", "content": "What's the weather?"} + assert transformed_messages[1]["role"] == "assistant" + assert transformed_messages[1].get("tool_calls") is not None + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json From 67f5ce9c7c23d476eeb371885b401c625ca61a40 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:41:26 +0530 Subject: [PATCH 7/9] address greptile review feedback (greploop iteration 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore backward compat for leading assistant(tool_calls) — always prepend user_continue - Replace partial assertions with full list assertion in trailing tool-call test Made-with: Cursor --- .../prompt_templates/common_utils.py | 8 +++--- tests/llm_translation/test_prompt_factory.py | 25 ++++++++++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index d7c1cc708b0..8713d0283ed 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -287,11 +287,9 @@ def _insert_user_continue_message( result_messages = messages.copy() # Don't modify the input list continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE - # Handle first counted message if it's an assistant message - if ( - result_messages[0]["role"] == "assistant" - and _counts_for_alternation(result_messages[0]) - ): + # Handle first message if it's an assistant message — always prepend + # user_continue regardless of tool_calls, to preserve backward compatibility. + if result_messages[0]["role"] == "assistant": result_messages.insert(0, continue_message) # Handle consecutive assistant messages in the counted sequence diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index b02ed3ebea1..3a9f267e6a0 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -929,12 +929,25 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant(): ensure_alternating_roles=True, ) - # Backward compat: trailing assistant (even with tool_calls) gets user_continue - # appended, then assistant_continue bridges the user→user gap. - assert transformed_messages[-1] == {"role": "user", "content": "Please continue."} - assert transformed_messages[0] == {"role": "user", "content": "What's the weather?"} - assert transformed_messages[1]["role"] == "assistant" - assert transformed_messages[1].get("tool_calls") is not None + assert transformed_messages == [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Please continue."}, + ] def test_alternating_roles_e2e(): From 22fc08d602598f5b5cbe2293ccbd146bb748e622 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 16:59:16 +0530 Subject: [PATCH 8/9] fix(prompting): revert _insert_assistant_continue_message to adjacent-check logic Restore backward-compatible behavior: only insert assistant_continue between directly adjacent user messages, not across tool-call chains. The _counts_for_alternation skip logic was a silent behavioral change for [user, assistant(tc), tool, user] sequences. Made-with: Cursor --- .../prompt_templates/common_utils.py | 42 +++++-------------- tests/llm_translation/test_prompt_factory.py | 3 +- 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 8713d0283ed..eb3755b71fe 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -326,45 +326,25 @@ def _insert_assistant_continue_message( ) -> List[AllMessageValues]: """ Add assistant continuation messages between consecutive user messages. - Skips tool messages and assistant messages with tool calls in the - alternation check, matching strict templates like llama.cpp. - Args: - messages: List of message dictionaries - assistant_continue_message: Optional custom assistant message - ensure_alternating_roles: Whether to enforce alternating roles - - Returns: - Modified list of messages with inserted assistant messages + Only checks directly adjacent messages to preserve backward compatibility. """ if not ensure_alternating_roles or len(messages) <= 1: return messages + continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE - insert_before_indexes = set() - - for i, message in enumerate(messages): - if message.get("role") != "user": - continue - - next_counted_index = i + 1 - while next_counted_index < len(messages) and not _counts_for_alternation( - messages[next_counted_index] - ): - next_counted_index += 1 - - if ( - next_counted_index < len(messages) - and messages[next_counted_index].get("role") == "user" - ): - # Insert before the next counted user turn. - # This avoids splitting assistant tool-call -> tool chains. - insert_before_indexes.add(next_counted_index) modified_messages: List[AllMessageValues] = [] - for idx, message in enumerate(messages): - if idx in insert_before_indexes: + for i, message in enumerate(messages): + if ( + i < len(messages) - 1 + and message.get("role") == "user" + and messages[i + 1].get("role") == "user" + ): + modified_messages.append(message) modified_messages.append(continue_message) - modified_messages.append(message) + else: + modified_messages.append(message) return modified_messages diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 3a9f267e6a0..fe46c24a298 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -860,6 +860,7 @@ def test_ensure_alternating_roles_three_consecutive_assistants(): def test_ensure_alternating_roles_does_not_split_tool_call_chain(): + """Tool-call chains [user, assistant(tc), tool, user] are preserved as-is.""" messages = [ {"role": "user", "content": "Search for X"}, { @@ -898,7 +899,6 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ], }, {"role": "tool", "tool_call_id": "c1", "content": "results"}, - {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Thanks, now do Y"}, ] @@ -945,7 +945,6 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant(): } ], }, - {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Please continue."}, ] From f29b4981a0fca42eb7c3918da1649a9036d74e49 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 18 Mar 2026 17:13:42 +0530 Subject: [PATCH 9/9] fix(prompting): preserve separator for assistant(tc)->assistant edge case When scanning backward over counted messages, preserve old behavior for adjacent assistant turns by inserting user_continue if the immediate previous raw message is assistant. This handles malformed assistant(tool_calls)->assistant(no-tool-calls) inputs without splitting valid assistant(tool_calls)->tool chains. Made-with: Cursor --- .../prompt_templates/common_utils.py | 27 +++++++---- tests/llm_translation/test_prompt_factory.py | 48 +++++++++++++++++++ 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index eb3755b71fe..2efd90e0c2f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -298,16 +298,23 @@ def _insert_user_continue_message( curr_message = result_messages[i] inserted_continue_message = False if _counts_for_alternation(curr_message) and curr_message["role"] == "assistant": - j = i - 1 - while j >= 0: - previous_message = result_messages[j] - if _counts_for_alternation(previous_message): - if previous_message["role"] == "assistant": - result_messages.insert(i, continue_message) - i += 2 - inserted_continue_message = True - break - j -= 1 + # Preserve old behavior for malformed adjacent assistant sequences like + # assistant(tool_calls) -> assistant(no-tool-calls) with no tool message. + if i > 0 and result_messages[i - 1].get("role") == "assistant": + result_messages.insert(i, continue_message) + i += 2 + inserted_continue_message = True + else: + j = i - 1 + while j >= 0: + previous_message = result_messages[j] + if _counts_for_alternation(previous_message): + if previous_message["role"] == "assistant": + result_messages.insert(i, continue_message) + i += 2 + inserted_continue_message = True + break + j -= 1 if not inserted_continue_message: i += 1 diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index fe46c24a298..64556c3f26d 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -903,6 +903,54 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ] +def test_ensure_alternating_roles_assistant_tool_call_then_assistant(): + """ + Preserve old behavior for malformed adjacent assistant turns: + [assistant(tool_calls), assistant(no-tool-calls), user] should insert + user_continue between assistant messages. + """ + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "assistant", "content": "Here's what I found."}, + {"role": "user", "content": "Thanks"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Please continue."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + } + ], + }, + {"role": "user", "content": "Please continue."}, + {"role": "assistant", "content": "Here's what I found."}, + {"role": "user", "content": "Thanks"}, + ] + + def test_ensure_alternating_roles_trailing_tool_call_assistant(): messages = [ {"role": "user", "content": "What's the weather?"},