diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index b3a64083b43..f01f1805a7a 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -1,3 +1,4 @@ +import asyncio import inspect from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping from types import MappingProxyType @@ -199,6 +200,9 @@ async def _native_amessages( args: tuple[object, ...], kwargs: Mapping[str, object], ) -> MessagesResult: + # Python's handler leaves the event loop (run_in_executor) before it builds the provider body, so + # tasks scheduled next to the call, such as the proxy's during_call guardrails, edit the request first + await asyncio.sleep(0) # Python's async @client wrapper stamps these on every failure; the proxy reads `timeout` for # the x-litellm-timeout header and the router reads `num_retries`. try: diff --git a/tests/test_litellm_rust/messages/test_requests.py b/tests/test_litellm_rust/messages/test_requests.py index 92031e72aee..42fc53d2c73 100644 --- a/tests/test_litellm_rust/messages/test_requests.py +++ b/tests/test_litellm_rust/messages/test_requests.py @@ -1,3 +1,4 @@ +import asyncio from typing import Final import pytest @@ -146,3 +147,20 @@ async def test_optional_positional_body_parameters_reach_the_wire(server: Record assert response["id"] == MESSAGES_RESPONSE["id"] body, _ = sent(server) assert (body["metadata"], body["system"], body["temperature"]) == (args[3], args[6], args[7]) + + +@pytest.mark.asyncio +async def test_in_place_edits_from_a_task_started_before_the_call_reach_the_wire(server: RecordingServer) -> None: + messages: Final = [{"role": "user", "content": "during about secret"}] + + async def mask() -> None: + messages[0]["content"] = "during about [REDACTED]" + + async def during_call() -> None: + await asyncio.gather(mask()) + + guardrail: Final = asyncio.create_task(during_call()) + await asyncio.gather(guardrail, call(server, messages=messages)) + + body, _ = sent(server) + assert body["messages"] == [{"role": "user", "content": "during about [REDACTED]"}]