mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(completion): keep assistant-only role-less turns and skip SDK and PagerDuty alerts
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
A role-less turn carrying any assistant-only key, null values included, is still filled as assistant, so a dumped assistant message or a reasoning turn keeps working. The Router's alerting_config alert and the PagerDuty failure count skip the role error, like the proxy alert already does.
This commit is contained in:
parent
64097cfea8
commit
3a8090486f
6 changed files with 100 additions and 6 deletions
|
|
@ -17,6 +17,7 @@ from typing import List, Optional, Union
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching import DualCache
|
||||
from litellm.exceptions import MissingMessageRoleError
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
|
|
@ -86,6 +87,8 @@ class PagerDutyAlerting(SlackAlerting):
|
|||
Record a failure event. Only send an alert to PagerDuty if the
|
||||
configured *failure* threshold is exceeded in the specified window.
|
||||
"""
|
||||
if isinstance(kwargs.get("exception"), MissingMessageRoleError):
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
|
||||
"standard_logging_object"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
from litellm._logging import redact_secrets, verbose_router_logger
|
||||
from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH
|
||||
from litellm.exceptions import MissingMessageRoleError
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
async_get_cooldown_deployments_with_debug_info,
|
||||
get_cooldown_deployment_info,
|
||||
|
|
@ -48,6 +49,9 @@ async def send_llm_exception_alert(
|
|||
if litellm_router_instance.slack_alerting_logger is None:
|
||||
return
|
||||
|
||||
if isinstance(original_exception, MissingMessageRoleError):
|
||||
return
|
||||
|
||||
if "proxy_server_request" in request_kwargs:
|
||||
# Do not send any alert if it's a request from litellm proxy server request
|
||||
# the proxy is already instrumented to send LLM API call failures
|
||||
|
|
|
|||
|
|
@ -9228,14 +9228,19 @@ def convert_list_message_to_dict(
|
|||
return [dict(cleanup_none_field_in_message(message=_as_message_value(message))) for message in messages]
|
||||
|
||||
|
||||
_ASSISTANT_ONLY_MESSAGE_KEYS: Final = frozenset(
|
||||
{"tool_calls", "function_call", "reasoning_content", "reasoning_items", "thinking_blocks", "refusal", "audio"}
|
||||
)
|
||||
|
||||
|
||||
def validate_and_fix_openai_messages(
|
||||
messages: list, model: str = "", request_kwargs: Mapping[str, object] | None = None
|
||||
):
|
||||
"""
|
||||
Ensures all messages are valid OpenAI chat completion messages.
|
||||
|
||||
Only an assistant turn carries tool_calls or function_call, so one sent without a role gets "assistant"
|
||||
(issue #8661).
|
||||
Only an assistant turn carries the keys in _ASSISTANT_ONLY_MESSAGE_KEYS, so one sent without a role but with
|
||||
any of them, even set to null, gets "assistant" (issue #8661).
|
||||
"""
|
||||
new_messages: Final = []
|
||||
for message in messages:
|
||||
|
|
@ -9249,7 +9254,7 @@ def validate_and_fix_openai_messages(
|
|||
),
|
||||
)
|
||||
cleaned_message = cleanup_none_field_in_message(message=convert_msg_to_dict)
|
||||
if not cleaned_message.get("role") and ("tool_calls" in cleaned_message or "function_call" in cleaned_message):
|
||||
if not cleaned_message.get("role") and not _ASSISTANT_ONLY_MESSAGE_KEYS.isdisjoint(convert_msg_to_dict):
|
||||
cleaned_message["role"] = "assistant"
|
||||
new_messages.append(cleaned_message)
|
||||
return validate_chat_completion_user_messages(messages=new_messages, model=model, request_kwargs=request_kwargs)
|
||||
|
|
|
|||
|
|
@ -74,6 +74,51 @@ async def test_failure_alerts_follow_the_configured_threshold(
|
|||
assert recent_errors[0]["error_llm_provider"] == "openai"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("exception", "request_count"),
|
||||
[
|
||||
(
|
||||
litellm.MissingMessageRoleError(
|
||||
message="Missing required parameter: 'messages[0].role'.", model="gpt-5.6", llm_provider="openai"
|
||||
),
|
||||
0,
|
||||
),
|
||||
(
|
||||
litellm.MalformedRequestFieldError(
|
||||
message="tools: Input should be a valid list", model="gpt-5.6", llm_provider="openai"
|
||||
),
|
||||
1,
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_only_a_role_less_request_stays_out_of_the_failure_count(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
exception: Exception,
|
||||
request_count: int,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(PAGERDUTY_URL).mock(return_value=httpx.Response(202))
|
||||
logger: Final = _pagerduty_logger(monkeypatch)
|
||||
payload: Final = cast(
|
||||
StandardLoggingPayload,
|
||||
{
|
||||
"error_information": {
|
||||
"error_class": type(exception).__name__,
|
||||
"error_code": "400",
|
||||
"llm_provider": "openai",
|
||||
},
|
||||
"metadata": {},
|
||||
},
|
||||
)
|
||||
|
||||
await logger.async_log_failure_event(
|
||||
{"standard_logging_object": payload, "exception": exception}, None, None, None
|
||||
)
|
||||
|
||||
assert route.call_count == request_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hanging_requests_alert_at_the_default_threshold(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import asyncio
|
||||
import importlib
|
||||
from typing import List
|
||||
from typing import Final, List
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -38,6 +38,36 @@ async def test_send_llm_exception_alert_success():
|
|||
mock_router.slack_alerting_logger.send_alert.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("exception", "alerts_sent"),
|
||||
[
|
||||
(
|
||||
litellm.MissingMessageRoleError(
|
||||
message="Missing required parameter: 'messages[0].role'.", model="gpt-5.6", llm_provider="openai"
|
||||
),
|
||||
0,
|
||||
),
|
||||
(
|
||||
litellm.MalformedRequestFieldError(
|
||||
message="tools: Input should be a valid list", model="gpt-5.6", llm_provider="openai"
|
||||
),
|
||||
1,
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_send_llm_exception_alert_skips_only_a_role_less_request(exception: Exception, alerts_sent: int):
|
||||
mock_router: Final = MagicMock()
|
||||
mock_router.slack_alerting_logger = AsyncMock()
|
||||
request_kwargs: Final = {"model": "gpt-5.6", "messages": [{"content": "Hello"}]}
|
||||
|
||||
from litellm.router_utils.handle_error import send_llm_exception_alert
|
||||
|
||||
await send_llm_exception_alert(mock_router, request_kwargs, "Traceback", exception)
|
||||
|
||||
assert mock_router.slack_alerting_logger.send_alert.call_count == alerts_sent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_llm_exception_alert_no_logger():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -8674,10 +8674,17 @@ def test_completion_message_without_role_is_a_400(messages: list[dict[str, str |
|
|||
},
|
||||
{"content": None, "function_call": {"name": "get_weather", "arguments": "{}"}},
|
||||
{"content": "ok", "tool_calls": []},
|
||||
{"content": "ok", "tool_calls": None},
|
||||
{"content": "ok", "function_call": None},
|
||||
{"content": "Sunny.", "reasoning_content": "The user wants the weather."},
|
||||
{"content": "Sunny.", "reasoning_items": []},
|
||||
{"content": "Sunny.", "thinking_blocks": [{"type": "thinking", "thinking": "Weather.", "signature": "sig"}]},
|
||||
{"content": None, "refusal": "I can't help with that."},
|
||||
{"content": None, "audio": {"id": "audio_1"}},
|
||||
],
|
||||
)
|
||||
def test_completion_role_less_tool_call_turn_is_sent_as_assistant(call_turn: dict[str, object]):
|
||||
"""Only an assistant turn carries tool calls, so one replayed without a role still goes upstream as assistant."""
|
||||
def test_completion_role_less_assistant_turn_is_sent_as_assistant(call_turn: dict[str, object]):
|
||||
"""Only an assistant turn carries these keys, so one replayed without a role still goes upstream as assistant."""
|
||||
messages: Final = [{"role": "user", "content": "Weather in Paris?"}, call_turn]
|
||||
|
||||
assert validate_and_fix_openai_messages(messages)[1]["role"] == "assistant"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue