mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge cf057ff182 into 61efbd071e
This commit is contained in:
commit
1a7e5b98d6
5 changed files with 235 additions and 39 deletions
|
|
@ -209,6 +209,10 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool:
|
|||
return media.part_type == "image_url"
|
||||
|
||||
|
||||
def inline_remote_file_urls(media: RemoteMedia) -> bool:
|
||||
return media.part_type == "file"
|
||||
|
||||
|
||||
def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None:
|
||||
if fields.get("type") != "image_url":
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -129,8 +129,6 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
|
||||
"""
|
||||
|
||||
_is_base_class = False
|
||||
|
||||
temperature: int | None = None
|
||||
top_p: int | None = None
|
||||
max_tokens: int | None = None
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Support for gpt model family
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
|
|
@ -32,7 +33,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_url_to_base64,
|
||||
async_inline_remote_media,
|
||||
convert_url_to_base64,
|
||||
inline_remote_file_urls,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
|
|
@ -163,9 +166,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
- `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling.
|
||||
"""
|
||||
|
||||
# Add a class variable to track if this is the base class
|
||||
_is_base_class = True
|
||||
|
||||
frequency_penalty: int | None = None
|
||||
function_call: str | dict | None = None
|
||||
functions: list | None = None
|
||||
|
|
@ -197,8 +197,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
self.__class__._is_base_class = False
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
return super().get_config()
|
||||
|
|
@ -646,29 +644,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect(
|
||||
model=model,
|
||||
messages=await self._transform_messages(
|
||||
messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True
|
||||
),
|
||||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
if self.__class__._is_base_class:
|
||||
if not optional_params.get("tools") and not optional_params.get("functions"):
|
||||
optional_params.pop("tool_choice", None)
|
||||
return {
|
||||
"model": model,
|
||||
"messages": transformed_messages,
|
||||
**optional_params,
|
||||
**self._sanitized_tools_update_for_openai(optional_params, litellm_params),
|
||||
}
|
||||
else:
|
||||
## allow for any object specific behaviour to be handled
|
||||
return self.transform_request(model, messages, optional_params, litellm_params, headers)
|
||||
try:
|
||||
downloaded: Final = await async_inline_remote_media(messages, should_inline=inline_remote_file_urls)
|
||||
except litellm.ImageFetchError:
|
||||
return await asyncio.to_thread(
|
||||
self.transform_request, model, messages, optional_params, litellm_params, headers
|
||||
)
|
||||
return self.transform_request(model, downloaded, optional_params, litellm_params, headers)
|
||||
|
||||
def _passed_in_tools(self, optional_params: dict) -> bool:
|
||||
return optional_params.get("tools", None) is not None
|
||||
|
|
|
|||
|
|
@ -1200,7 +1200,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
shared_session: Optional["ClientSession"] = None,
|
||||
):
|
||||
response = None
|
||||
data = provider_config.transform_request(
|
||||
data = await provider_config.async_transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
"""
|
||||
Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py)
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import copy
|
||||
import itertools
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
from httpx import Response
|
||||
from httpx import Request, Response
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -17,11 +19,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
TOOL_RESULT_MEDIA_BOUNDARY,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache
|
||||
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
|
||||
from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config
|
||||
from litellm.llms.openai.chat.gpt_transformation import (
|
||||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from litellm.llms.sambanova.chat import SambanovaConfig
|
||||
|
||||
|
||||
class TestOpenAIGPTConfig:
|
||||
|
|
@ -1348,6 +1352,220 @@ async def test_openai_media_download_preserves_supported_chat_fields(
|
|||
|
||||
|
||||
|
||||
_PDF_BYTES: Final = b"%PDF-1.7\nmock-pdf"
|
||||
_DOWNLOADED_PDF_PART: Final = {
|
||||
"type": "file",
|
||||
"file": {
|
||||
"file_data": f"data:application/pdf;base64,{base64.b64encode(_PDF_BYTES).decode('utf-8')}",
|
||||
"filename": "my_file.pdf",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _pdf_url_part(url: str) -> dict[str, object]:
|
||||
return {"type": "file", "file": {"file_id": url}}
|
||||
|
||||
|
||||
async def _serve_pdf_to_async_clients_only(request: Request) -> Response:
|
||||
return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES)
|
||||
|
||||
|
||||
def _tool_result_messages(*content: object) -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "user", "content": "read the report"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "fetch_report", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": list(content)},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_urls_reachable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
|
||||
_SUMMARIZE: Final = {"type": "text", "text": "Summarize the report."}
|
||||
_UPLOADED_FILE_PART: Final = {"type": "file", "file": {"file_id": "file-uploaded"}}
|
||||
_ONCE_URL: Final = "https://files.example.test/download-once.pdf"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("messages", "expected_messages"),
|
||||
[
|
||||
pytest.param(
|
||||
[{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _pdf_url_part(_ONCE_URL)]}],
|
||||
[{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _DOWNLOADED_PDF_PART]}],
|
||||
id="user_message",
|
||||
),
|
||||
pytest.param(
|
||||
_tool_result_messages(_SUMMARIZE, _pdf_url_part(_ONCE_URL)),
|
||||
[
|
||||
*_tool_result_messages(_SUMMARIZE)[:2],
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": [_SUMMARIZE]},
|
||||
{"role": "user", "content": [{"type": "text", "text": TOOL_RESULT_MEDIA_BOUNDARY}, _DOWNLOADED_PDF_PART]},
|
||||
],
|
||||
id="tool_result",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_async_transform_request_downloads_a_pdf_url_once_and_leaves_the_callers_messages_unchanged(
|
||||
respx_mock: respx.MockRouter, messages: list[dict[str, object]], expected_messages: list[dict[str, object]]
|
||||
) -> None:
|
||||
in_memory_cache.delete_cache(_ONCE_URL)
|
||||
download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only)
|
||||
sent: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=sent,
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "openai"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert download.call_count == 1
|
||||
assert request["messages"] == expected_messages
|
||||
assert sent == messages
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_downloads_the_pdf_urls_of_one_request_at_the_same_time(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
pdf_urls: Final = tuple(f"https://files.example.test/together-{index}.pdf" for index in range(3))
|
||||
for pdf_url in pdf_urls:
|
||||
in_memory_cache.delete_cache(pdf_url)
|
||||
started_downloads: Final = itertools.count(1)
|
||||
every_download_started: Final = asyncio.Event()
|
||||
|
||||
async def serve_pdf_once_every_download_started(request: Request) -> Response:
|
||||
if next(started_downloads) == len(pdf_urls):
|
||||
every_download_started.set()
|
||||
try:
|
||||
await asyncio.wait_for(every_download_started.wait(), timeout=5)
|
||||
except TimeoutError as error:
|
||||
raise litellm.ImageFetchError("the PDF URLs were downloaded one after another") from error
|
||||
return await _serve_pdf_to_async_clients_only(request)
|
||||
|
||||
respx_mock.route(method="GET", host="files.example.test").mock(side_effect=serve_pdf_once_every_download_started)
|
||||
|
||||
request: Final = await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE, *map(_pdf_url_part, pdf_urls)]}],
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "openai"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, *[_DOWNLOADED_PDF_PART] * 3]}]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_completion_downloads_its_pdf_url_with_the_async_client(respx_mock: respx.MockRouter) -> None:
|
||||
pdf_url: Final = "https://files.example.test/streamed.pdf"
|
||||
in_memory_cache.delete_cache(pdf_url)
|
||||
download: Final = respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only)
|
||||
chunk: Final = {
|
||||
"id": "chatcmpl-stream",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "gpt-4.1",
|
||||
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "seen"}, "finish_reason": "stop"}],
|
||||
}
|
||||
chat: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode(),
|
||||
)
|
||||
)
|
||||
|
||||
stream: Final = await litellm.acompletion(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}],
|
||||
api_key="sk-openai-test",
|
||||
stream=True,
|
||||
)
|
||||
text: Final = "".join([part.choices[0].delta.content or "" async for part in stream])
|
||||
|
||||
assert text == "seen"
|
||||
assert download.call_count == 1
|
||||
assert json.loads(chat.calls[0].request.content)["messages"] == [
|
||||
{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_rejects_a_file_part_without_its_file_and_leaves_the_callers_messages_unchanged(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
pdf_url: Final = "https://files.example.test/before-a-broken-part.pdf"
|
||||
in_memory_cache.delete_cache(pdf_url)
|
||||
respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only)
|
||||
messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url), {"type": "file", "file": None}]}]
|
||||
sent: Final = copy.deepcopy(messages)
|
||||
|
||||
with pytest.raises(
|
||||
litellm.BadRequestError, match="Content block has type='file' but is missing the required 'file' field"
|
||||
):
|
||||
await OpenAIGPTConfig().async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=sent,
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "openai"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sent == messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("config", [OpenAIGPTConfig(), DeepInfraConfig()], ids=["openai", "deepinfra"])
|
||||
async def test_async_transform_request_sends_a_plain_string_tool_result_part_like_the_sync_transform(
|
||||
config: OpenAIGPTConfig,
|
||||
) -> None:
|
||||
messages: Final = _tool_result_messages("plain string result")
|
||||
|
||||
request: Final = await config.async_transform_request(
|
||||
model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert request == config.transform_request(
|
||||
model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("pdf_urls_reachable")
|
||||
@pytest.mark.respx(assert_all_called=False)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_still_flattens_for_a_provider_without_files_when_the_pdf_url_fails(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
pdf_url: Final = "https://files.example.test/missing.pdf"
|
||||
in_memory_cache.delete_cache(pdf_url)
|
||||
respx_mock.get(pdf_url).mock(return_value=Response(404))
|
||||
messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}]
|
||||
|
||||
request: Final = await SambanovaConfig().async_transform_request(
|
||||
model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert request["messages"] == [{"role": "user", "content": "Summarize the report."}]
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_vision_with_custom_model_preserves_base64_image_in_request(respx_mock: respx.MockRouter) -> None:
|
||||
image_uri: Final = f"data:image/png;base64,{base64.b64encode(b'test-image').decode('utf-8')}"
|
||||
|
|
@ -1560,14 +1778,8 @@ class TestSystemMessagesFirst:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transform_request_moves_system_messages_first(self, monkeypatch):
|
||||
class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig):
|
||||
_is_base_class = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(litellm, "openai_system_messages_first", True)
|
||||
request = await UninstantiatedOpenAIGPTConfig().async_transform_request(
|
||||
request = await self.config.async_transform_request(
|
||||
model="gpt-4.1",
|
||||
messages=self._messages(),
|
||||
optional_params={},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue