From cd64d3252fc91b38cfdaf6b9bc58121c6b1767ae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:35:53 -0700 Subject: [PATCH 1/2] fix(openai): transform async chat messages once, whatever configs were built before --- .../llms/openai/chat/gpt_transformation.py | 42 ++++----- .../chat/test_openai_gpt_transformation.py | 88 +++++++++++++++++-- 2 files changed, 95 insertions(+), 35 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 22af51aaabe..e64033e143b 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -4,7 +4,9 @@ Support for gpt model family import json import os +from asyncio import to_thread from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload from urllib.parse import urlparse @@ -136,6 +138,14 @@ def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | Non return extra_body.get("prompt_cache_options") +def _attached_files(message: AllMessageValues) -> Iterator[ChatCompletionFileObjectFile]: + match message: + case {"role": "user", "content": list() as content}: + return (part["file"] for part in content if part.get("type") == "file" and "file" in part) + case _: + return iter(()) + + class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): """ Reference: https://platform.openai.com/docs/api-reference/chat/create @@ -163,9 +173,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. """ - # Add a class variable to track if this is the base class - _is_base_class = True - frequency_penalty: int | None = None function_call: str | dict | None = None functions: list | None = None @@ -197,8 +204,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) - self.__class__._is_base_class = False - @classmethod def get_config(cls): return super().get_config() @@ -646,29 +651,12 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect( - model=model, - messages=await self._transform_messages( - messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True - ), - tools=optional_params.get("tools", []), - prompt_cache_options=_prompt_cache_options(optional_params), - litellm_params=litellm_params, - ) - if tools is not None and len(tools) > 0: - optional_params["tools"] = tools - if self.__class__._is_base_class: - if not optional_params.get("tools") and not optional_params.get("functions"): - optional_params.pop("tool_choice", None) - return { - "model": model, - "messages": transformed_messages, - **optional_params, - **self._sanitized_tools_update_for_openai(optional_params, litellm_params), - } - else: - ## allow for any object specific behaviour to be handled + if not self._downloads_pdf_url(messages): return self.transform_request(model, messages, optional_params, litellm_params, headers) + return await to_thread(self.transform_request, model, messages, optional_params, litellm_params, headers) + + def _downloads_pdf_url(self, messages: Sequence[AllMessageValues]) -> bool: + return any(self.contains_pdf_url(file) for file in chain.from_iterable(map(_attached_files, messages))) def _passed_in_tools(self, optional_params: dict) -> bool: return optional_params.get("tools", None) is not None diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 5b85a94b24d..2b93aada819 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1,10 +1,12 @@ """ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ +import asyncio import base64 import copy import json -from collections.abc import Callable +import threading +from collections.abc import Callable, Coroutine from typing import Final import pytest @@ -1348,6 +1350,82 @@ async def test_openai_media_download_preserves_supported_chat_fields( +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_codestral_file_request_still_works_after_an_openai_config_is_built( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chat: Final = respx_mock.post("https://codestral.example.test/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + OpenAIGPTConfig() + + await litellm.acompletion( + model="codestral/codestral-latest", + api_base="https://codestral.example.test/v1", + api_key="mistral-test-key", + messages=[{"role": "user", "content": [ + {"type": "text", "text": "What is in the file?"}, + {"type": "file", "file": {"file_id": "uploaded-file-id"}}, + ]}], + ) + + assert json.loads(chat.calls[0].request.content)["messages"][0]["content"][1] == { + "type": "file", "file_id": "uploaded-file-id", + } + + +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_event_loop( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + pdf_url: Final = "https://files.example.test/off-event-loop.pdf" + pdf_content: Final = b"%PDF-1.7\nmock-pdf" + in_memory_cache.delete_cache(pdf_url) + event_loop_thread: Final = threading.current_thread() + event_loop_ran: Final = threading.Event() + loop_ran_during_download: Final[list[bool]] = [] + + def pdf_response() -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=pdf_content) + + async def serve_pdf_on_event_loop() -> Response: + loop_ran_during_download.append(await asyncio.to_thread(event_loop_ran.wait, 5)) + return pdf_response() + + def serve_pdf(request: object) -> Response | Coroutine[object, object, Response]: + if threading.current_thread() is event_loop_thread: + return serve_pdf_on_event_loop() + loop_ran_during_download.append(event_loop_ran.wait(timeout=5)) + return pdf_response() + + respx_mock.get(pdf_url).mock(side_effect=serve_pdf) + + async def mark_event_loop_ran() -> None: + event_loop_ran.set() + + request, _ = await asyncio.gather( + OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": pdf_url}}]}], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ), + mark_event_loop_ran(), + ) + + assert loop_ran_during_download == [True] + assert request["messages"][0]["content"][0]["file"] == { + "file_data": f"data:application/pdf;base64,{base64.b64encode(pdf_content).decode('utf-8')}", + "filename": "my_file.pdf", + } + + @pytest.mark.respx(assert_all_called=True) def test_vision_with_custom_model_preserves_base64_image_in_request(respx_mock: respx.MockRouter) -> None: image_uri: Final = f"data:image/png;base64,{base64.b64encode(b'test-image').decode('utf-8')}" @@ -1560,14 +1638,8 @@ class TestSystemMessagesFirst: @pytest.mark.asyncio async def test_async_transform_request_moves_system_messages_first(self, monkeypatch): - class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig): - _is_base_class = True - - def __init__(self) -> None: - pass - monkeypatch.setattr(litellm, "openai_system_messages_first", True) - request = await UninstantiatedOpenAIGPTConfig().async_transform_request( + request = await self.config.async_transform_request( model="gpt-4.1", messages=self._messages(), optional_params={}, From acb9f0abb15b648e92b2e685614c0bdcc2521e00 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:13:35 -0700 Subject: [PATCH 2/2] fix(openai): keep tool-result PDF downloads off the event loop and stop Mistral rewriting retried messages --- litellm/llms/mistral/chat/transformation.py | 32 +++++++++-------- .../llms/openai/chat/gpt_transformation.py | 11 ++++-- .../test_mistral_chat_transformation.py | 28 +++++++++++++++ .../chat/test_openai_gpt_transformation.py | 36 +++++++++++++++---- 4 files changed, 84 insertions(+), 23 deletions(-) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 2fb9402ed03..8b14ae126ad 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -63,6 +63,23 @@ def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a de return isinstance(value, list) +def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]: + match block: + case {"type": "file", "file": {"file_id": file_id}} if file_id: + return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id} + case _: + return block + + +def _with_mistral_file_ids(message: AllMessageValues) -> AllMessageValues: + content: Final = message.get("content") + if not isinstance(content, list): + return message + return cast( # cast-ok: dict spread keeps message shape + AllMessageValues, {**message, "content": [_with_mistral_file_id(block) for block in content]} + ) + + class MistralConfig(OpenAIGPTConfig): """ Reference: https://docs.mistral.ai/api/ @@ -363,20 +380,7 @@ class MistralConfig(OpenAIGPTConfig): """ Mistral API supports only 'file_id' in message content with type 'file'. """ - for m in messages: - _content_block = m.get("content") - if _content_block and isinstance(_content_block, list): - if any(c.get("type") == "file" for c in _content_block): - # If file content is present, we get file_id from 'file' attribute of content block - # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it. - file_contents = [c for c in _content_block if c.get("type") == "file"] - for file_content in file_contents: - file_id = file_content.get("file", {}).get("file_id") - if file_id: - # Replace 'file' with 'file_id' - file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape - file_content.pop("file", None) - return messages + return [_with_mistral_file_ids(message) for message in messages] def _add_reasoning_system_prompt_if_needed( self, messages: list[AllMessageValues], optional_params: dict diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index e64033e143b..ad3f12a1486 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -4,8 +4,11 @@ Support for gpt model family import json import os -from asyncio import to_thread +from asyncio import get_running_loop from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context +from functools import partial from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload @@ -81,6 +84,7 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) +_PDF_URL_TRANSFORM_EXECUTOR: Final = ThreadPoolExecutor(thread_name_prefix="litellm-pdf-url-transform") def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: @@ -140,7 +144,7 @@ def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | Non def _attached_files(message: AllMessageValues) -> Iterator[ChatCompletionFileObjectFile]: match message: - case {"role": "user", "content": list() as content}: + case {"role": "user" | "tool", "content": list() as content}: return (part["file"] for part in content if part.get("type") == "file" and "file" in part) case _: return iter(()) @@ -653,7 +657,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) -> dict: if not self._downloads_pdf_url(messages): return self.transform_request(model, messages, optional_params, litellm_params, headers) - return await to_thread(self.transform_request, model, messages, optional_params, litellm_params, headers) + transform: Final = partial(self.transform_request, model, messages, optional_params, litellm_params, headers) + return await get_running_loop().run_in_executor(_PDF_URL_TRANSFORM_EXECUTOR, copy_context().run, transform) def _downloads_pdf_url(self, messages: Sequence[AllMessageValues]) -> bool: return any(self.contains_pdf_url(file) for file in chain.from_iterable(map(_attached_files, messages))) diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index b76e1707805..13683ee6a78 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -799,6 +799,34 @@ class TestMistralFileHandling: assert result[0]["content"][2]["file_id"] == "file-67890" # type: ignore +@pytest.mark.asyncio +async def test_mistral_sends_an_uploaded_file_again_when_the_same_messages_are_retried(): + messages: Final = cast( + list[AllMessageValues], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize the attached file."}, + {"type": "file", "file": {"file_id": "file-12345"}}, + ], + } + ], + ) + config: Final = MistralConfig() + + requests: Final = [ + await config.async_transform_request( + model="codestral-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + for _ in range(2) + ] + + assert [request["messages"][0]["content"][1] for request in requests] == [ + {"type": "file", "file_id": "file-12345"} + ] * 2 + + class TestMistralStripsOutputOnlyFields: """Mistral rejects unknown input fields with a 422 ``extra_forbidden``. diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 2b93aada819..9ab4ca66ba8 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -24,6 +24,7 @@ from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.types.llms.openai import AllMessageValues class TestOpenAIGPTConfig: @@ -1376,16 +1377,39 @@ async def test_codestral_file_request_still_works_after_an_openai_config_is_buil } +_PDF_URL: Final = "https://files.example.test/off-event-loop.pdf" +_PDF_URL_PART: Final = {"type": "file", "file": {"file_id": _PDF_URL}} + + @pytest.mark.respx(assert_all_called=True) @pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param([{"role": "user", "content": [_PDF_URL_PART]}], id="user_message"), + pytest.param( + [ + {"role": "user", "content": "read the report"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fetch_report", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [_PDF_URL_PART]}, + ], + id="tool_result", + ), + ], +) async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_event_loop( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, messages: list[AllMessageValues] ) -> None: monkeypatch.setattr(litellm, "user_url_validation", False) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - pdf_url: Final = "https://files.example.test/off-event-loop.pdf" pdf_content: Final = b"%PDF-1.7\nmock-pdf" - in_memory_cache.delete_cache(pdf_url) + in_memory_cache.delete_cache(_PDF_URL) event_loop_thread: Final = threading.current_thread() event_loop_ran: Final = threading.Event() loop_ran_during_download: Final[list[bool]] = [] @@ -1403,7 +1427,7 @@ async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_e loop_ran_during_download.append(event_loop_ran.wait(timeout=5)) return pdf_response() - respx_mock.get(pdf_url).mock(side_effect=serve_pdf) + respx_mock.get(_PDF_URL).mock(side_effect=serve_pdf) async def mark_event_loop_ran() -> None: event_loop_ran.set() @@ -1411,7 +1435,7 @@ async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_e request, _ = await asyncio.gather( OpenAIGPTConfig().async_transform_request( model="gpt-4.1", - messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": pdf_url}}]}], + messages=copy.deepcopy(messages), optional_params={}, litellm_params={"custom_llm_provider": "openai"}, headers={}, @@ -1420,7 +1444,7 @@ async def test_async_transform_request_downloads_pdf_urls_without_blocking_the_e ) assert loop_ran_during_download == [True] - assert request["messages"][0]["content"][0]["file"] == { + assert request["messages"][-1]["content"][-1]["file"] == { "file_data": f"data:application/pdf;base64,{base64.b64encode(pdf_content).decode('utf-8')}", "filename": "my_file.pdf", }