diff --git a/litellm/llms/openai/completion/transformation.py b/litellm/llms/openai/completion/transformation.py index f7d2ce68082..a3bee682725 100644 --- a/litellm/llms/openai/completion/transformation.py +++ b/litellm/llms/openai/completion/transformation.py @@ -13,6 +13,34 @@ from litellm.types.utils import Choices, Message, ModelResponse, TextCompletionR from ..chat.gpt_transformation import OpenAIGPTConfig from .utils import transform_prompt +OPENAI_TEXT_COMPLETION_PARAMS: Final = frozenset( + { + "model", + "prompt", + "best_of", + "echo", + "frequency_penalty", + "logit_bias", + "logprobs", + "max_tokens", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "suffix", + "temperature", + "top_p", + "user", + "extra_headers", + "extra_query", + "extra_body", + "timeout", + "max_retries", + } +) + class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): """ @@ -155,8 +183,18 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): headers: dict, ) -> dict: prompt: Final = transform_prompt(messages) - return { + result: Final[dict[str, object]] = { # mutable-ok: completion request payload "model": model, "prompt": prompt, - **optional_params, } + result.update(optional_params) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # optional_params is an untyped dict + extra_body: Final[dict[str, object]] = {} # mutable-ok: completion extra body + user_extra: Final = result.pop("extra_body", None) + if isinstance(user_extra, (dict, Mapping)): + extra_body.update(user_extra) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # extra_body is untyped + for param in list(result.keys()): + if param not in OPENAI_TEXT_COMPLETION_PARAMS: + extra_body[param] = result.pop(param) + if extra_body: + result["extra_body"] = extra_body + return result diff --git a/tests/unit/llms/openai/completion/test_completion_handler.py b/tests/unit/llms/openai/completion/test_completion_handler.py index 628c9bbd783..abb9319ddf3 100644 --- a/tests/unit/llms/openai/completion/test_completion_handler.py +++ b/tests/unit/llms/openai/completion/test_completion_handler.py @@ -5,7 +5,6 @@ text completion path. Regression tests for https://github.com/BerriAI/litellm/issues/27410 """ - import json from typing import Final @@ -13,7 +12,6 @@ import pytest import respx from httpx import Response - import litellm from litellm import atext_completion, text_completion @@ -78,9 +76,7 @@ def test_completion_forwards_extra_headers_to_provider(mock_completions_endpoint @respx.mock -async def test_acompletion_forwards_client_headers_to_provider( - mock_completions_endpoint, monkeypatch -): +async def test_acompletion_forwards_client_headers_to_provider(mock_completions_endpoint, monkeypatch): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) await atext_completion( model="gpt-3.5-turbo-instruct", @@ -305,11 +301,9 @@ async def test_async_text_completion_stream_returns_text_and_one_finish_reason(m chunks: Final = [chunk async for chunk in stream] assert [chunk.choices[0].text for chunk in chunks if chunk.choices[0].text] == ["hello"] - assert tuple( - chunk.choices[0].finish_reason - for chunk in chunks - if chunk.choices[0].finish_reason is not None - ) == ("stop",) + assert tuple(chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason is not None) == ( + "stop", + ) assert len(route.calls) == 1 @@ -328,9 +322,7 @@ async def test_async_text_completion_chat_model_stream_builds_chat_chunks(monkey ) ) - stream: Final = await atext_completion( - model="gpt-3.5-turbo", prompt="say hello", stream=True, max_tokens=5 - ) + stream: Final = await atext_completion(model="gpt-3.5-turbo", prompt="say hello", stream=True, max_tokens=5) chunks: Final = [chunk async for chunk in stream] response: Final = litellm.stream_chunk_builder(chunks=chunks) @@ -340,11 +332,9 @@ async def test_async_text_completion_chat_model_stream_builds_chat_chunks(monkey completion_response=response, custom_cost_per_token={"input_cost_per_token": 5e-07, "output_cost_per_token": 1.5e-06}, ) == pytest.approx(1 * 1.5e-06) - assert tuple( - chunk.choices[0].finish_reason - for chunk in chunks - if chunk.choices[0].finish_reason is not None - ) == ("stop",) + assert tuple(chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason is not None) == ( + "stop", + ) assert len(route.calls) == 1 @@ -380,6 +370,54 @@ def test_text_completion_stream_forwards_stream_options_and_usage(monkeypatch): 1, 3, ) - assert json.loads(route.calls[0].request.content)["stream_options"] == { - "include_usage": True + assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} + + +@respx.mock +def test_completion_forwards_response_format_via_extra_body(mock_completions_endpoint): + text_completion( + model="gpt-3.5-turbo-instruct", + prompt="say hello", + response_format={"type": "json_object"}, + ) + + request_body = json.loads(mock_completions_endpoint.calls.last.request.content) + assert request_body["response_format"] == {"type": "json_object"} + + +@respx.mock +async def test_acompletion_forwards_response_format_via_extra_body(mock_completions_endpoint, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + await atext_completion( + model="gpt-3.5-turbo-instruct", + prompt="say hello", + response_format={"type": "json_object"}, + ) + + request_body = json.loads(mock_completions_endpoint.calls.last.request.content) + assert request_body["response_format"] == {"type": "json_object"} + + +def test_transform_text_completion_request_preserves_standard_and_moves_extra_to_extra_body(): + from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig + + config = OpenAITextCompletionConfig() + result = config.transform_text_completion_request( + model="gpt-3.5-turbo-instruct", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "temperature": 0.7, + "response_format": {"type": "json_object"}, + "extra_body": {"custom_option": 123}, + }, + headers={}, + ) + + assert result["model"] == "gpt-3.5-turbo-instruct" + assert result["prompt"] == "hello" + assert result["temperature"] == 0.7 + assert "response_format" not in result # moved to extra_body + assert result["extra_body"] == { + "custom_option": 123, + "response_format": {"type": "json_object"}, }