diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 3287515c81d..f458661241f 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -209,6 +209,10 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool: return media.part_type == "image_url" +def inline_remote_file_urls(media: RemoteMedia) -> bool: + return media.part_type == "file" + + def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None: if fields.get("type") != "image_url": return None diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index db8dd3fa7f0..4093723ed7b 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -129,8 +129,6 @@ class MistralConfig(OpenAIGPTConfig): - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. """ - _is_base_class = False - temperature: int | None = None top_p: int | None = None max_tokens: int | None = None diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 22af51aaabe..e152d56e470 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -2,6 +2,7 @@ Support for gpt model family """ +import asyncio import json import os from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence @@ -32,7 +33,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, + async_inline_remote_media, convert_url_to_base64, + inline_remote_file_urls, ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator @@ -163,9 +166,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. """ - # Add a class variable to track if this is the base class - _is_base_class = True - frequency_penalty: int | None = None function_call: str | dict | None = None functions: list | None = None @@ -197,8 +197,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) - self.__class__._is_base_class = False - @classmethod def get_config(cls): return super().get_config() @@ -646,29 +644,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect( - model=model, - messages=await self._transform_messages( - messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True - ), - tools=optional_params.get("tools", []), - prompt_cache_options=_prompt_cache_options(optional_params), - litellm_params=litellm_params, - ) - if tools is not None and len(tools) > 0: - optional_params["tools"] = tools - if self.__class__._is_base_class: - if not optional_params.get("tools") and not optional_params.get("functions"): - optional_params.pop("tool_choice", None) - return { - "model": model, - "messages": transformed_messages, - **optional_params, - **self._sanitized_tools_update_for_openai(optional_params, litellm_params), - } - else: - ## allow for any object specific behaviour to be handled - return self.transform_request(model, messages, optional_params, litellm_params, headers) + try: + downloaded: Final = await async_inline_remote_media(messages, should_inline=inline_remote_file_urls) + except litellm.ImageFetchError: + return await asyncio.to_thread( + self.transform_request, model, messages, optional_params, litellm_params, headers + ) + return self.transform_request(model, downloaded, optional_params, litellm_params, headers) def _passed_in_tools(self, optional_params: dict) -> bool: return optional_params.get("tools", None) is not None diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index bdd9f285823..dfc1ae155d9 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1200,7 +1200,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): shared_session: Optional["ClientSession"] = None, ): response = None - data = provider_config.transform_request( + data = await provider_config.async_transform_request( model=model, messages=messages, optional_params=optional_params, diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 5b85a94b24d..f88f1829cb6 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1,15 +1,17 @@ """ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ +import asyncio import base64 import copy +import itertools import json from collections.abc import Callable from typing import Final import pytest import respx -from httpx import Response +from httpx import Request, Response import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -17,11 +19,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_MEDIA_BOUNDARY, ) from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache +from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.llms.sambanova.chat import SambanovaConfig class TestOpenAIGPTConfig: @@ -1348,6 +1352,220 @@ async def test_openai_media_download_preserves_supported_chat_fields( +_PDF_BYTES: Final = b"%PDF-1.7\nmock-pdf" +_DOWNLOADED_PDF_PART: Final = { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{base64.b64encode(_PDF_BYTES).decode('utf-8')}", + "filename": "my_file.pdf", + }, +} + + +def _pdf_url_part(url: str) -> dict[str, object]: + return {"type": "file", "file": {"file_id": url}} + + +async def _serve_pdf_to_async_clients_only(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + +def _tool_result_messages(*content: object) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "read the report"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "fetch_report", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": list(content)}, + ] + + +@pytest.fixture +def pdf_urls_reachable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +_SUMMARIZE: Final = {"type": "text", "text": "Summarize the report."} +_UPLOADED_FILE_PART: Final = {"type": "file", "file": {"file_id": "file-uploaded"}} +_ONCE_URL: Final = "https://files.example.test/download-once.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("messages", "expected_messages"), + [ + pytest.param( + [{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _pdf_url_part(_ONCE_URL)]}], + [{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _DOWNLOADED_PDF_PART]}], + id="user_message", + ), + pytest.param( + _tool_result_messages(_SUMMARIZE, _pdf_url_part(_ONCE_URL)), + [ + *_tool_result_messages(_SUMMARIZE)[:2], + {"role": "tool", "tool_call_id": "call_1", "content": [_SUMMARIZE]}, + {"role": "user", "content": [{"type": "text", "text": TOOL_RESULT_MEDIA_BOUNDARY}, _DOWNLOADED_PDF_PART]}, + ], + id="tool_result", + ), + ], +) +async def test_async_transform_request_downloads_a_pdf_url_once_and_leaves_the_callers_messages_unchanged( + respx_mock: respx.MockRouter, messages: list[dict[str, object]], expected_messages: list[dict[str, object]] +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + sent: Final = copy.deepcopy(messages) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=sent, + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"] == expected_messages + assert sent == messages + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_the_same_time( + respx_mock: respx.MockRouter, +) -> None: + pdf_urls: Final = tuple(f"https://files.example.test/together-{index}.pdf" for index in range(3)) + for pdf_url in pdf_urls: + in_memory_cache.delete_cache(pdf_url) + started_downloads: Final = itertools.count(1) + every_download_started: Final = asyncio.Event() + + async def serve_pdf_once_every_download_started(request: Request) -> Response: + if next(started_downloads) == len(pdf_urls): + every_download_started.set() + try: + await asyncio.wait_for(every_download_started.wait(), timeout=5) + except TimeoutError as error: + raise litellm.ImageFetchError("the PDF URLs were downloaded one after another") from error + return await _serve_pdf_to_async_clients_only(request) + + respx_mock.route(method="GET", host="files.example.test").mock(side_effect=serve_pdf_once_every_download_started) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, *map(_pdf_url_part, pdf_urls)]}], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, *[_DOWNLOADED_PDF_PART] * 3]}] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_streamed_completion_downloads_its_pdf_url_with_the_async_client(respx_mock: respx.MockRouter) -> None: + pdf_url: Final = "https://files.example.test/streamed.pdf" + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) + chunk: Final = { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "created": 0, + "model": "gpt-4.1", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "seen"}, "finish_reason": "stop"}], + } + chat: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode(), + ) + ) + + stream: Final = await litellm.acompletion( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + api_key="sk-openai-test", + stream=True, + ) + text: Final = "".join([part.choices[0].delta.content or "" async for part in stream]) + + assert text == "seen" + assert download.call_count == 1 + assert json.loads(chat.calls[0].request.content)["messages"] == [ + {"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]} + ] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_rejects_a_file_part_without_its_file_and_leaves_the_callers_messages_unchanged( + respx_mock: respx.MockRouter, +) -> None: + pdf_url: Final = "https://files.example.test/before-a-broken-part.pdf" + in_memory_cache.delete_cache(pdf_url) + respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url), {"type": "file", "file": None}]}] + sent: Final = copy.deepcopy(messages) + + with pytest.raises( + litellm.BadRequestError, match="Content block has type='file' but is missing the required 'file' field" + ): + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=sent, + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert sent == messages + + +@pytest.mark.asyncio +@pytest.mark.parametrize("config", [OpenAIGPTConfig(), DeepInfraConfig()], ids=["openai", "deepinfra"]) +async def test_async_transform_request_sends_a_plain_string_tool_result_part_like_the_sync_transform( + config: OpenAIGPTConfig, +) -> None: + messages: Final = _tool_result_messages("plain string result") + + request: Final = await config.async_transform_request( + model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert request == config.transform_request( + model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +async def test_async_transform_request_still_flattens_for_a_provider_without_files_when_the_pdf_url_fails( + respx_mock: respx.MockRouter, +) -> None: + pdf_url: Final = "https://files.example.test/missing.pdf" + in_memory_cache.delete_cache(pdf_url) + respx_mock.get(pdf_url).mock(return_value=Response(404)) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}] + + request: Final = await SambanovaConfig().async_transform_request( + model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": "Summarize the report."}] + + @pytest.mark.respx(assert_all_called=True) def test_vision_with_custom_model_preserves_base64_image_in_request(respx_mock: respx.MockRouter) -> None: image_uri: Final = f"data:image/png;base64,{base64.b64encode(b'test-image').decode('utf-8')}" @@ -1560,14 +1778,8 @@ class TestSystemMessagesFirst: @pytest.mark.asyncio async def test_async_transform_request_moves_system_messages_first(self, monkeypatch): - class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig): - _is_base_class = True - - def __init__(self) -> None: - pass - monkeypatch.setattr(litellm, "openai_system_messages_first", True) - request = await UninstantiatedOpenAIGPTConfig().async_transform_request( + request = await self.config.async_transform_request( model="gpt-4.1", messages=self._messages(), optional_params={},