diff --git a/litellm/llms/openai/completion/transformation.py b/litellm/llms/openai/completion/transformation.py index f7d2ce68082..28c8e1372fb 100644 --- a/litellm/llms/openai/completion/transformation.py +++ b/litellm/llms/openai/completion/transformation.py @@ -5,6 +5,8 @@ Support for gpt model family from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter + from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage @@ -13,6 +15,13 @@ from litellm.types.utils import Choices, Message, ModelResponse, TextCompletionR from ..chat.gpt_transformation import OpenAIGPTConfig from .utils import transform_prompt +# Supported params the SDK's ``completions.create()`` has no keyword for. They go in ``extra_body`` +# so backends that accept them (e.g. vLLM) receive them, instead of the SDK raising a TypeError. +_BODY_ONLY_PARAMS: Final = frozenset( + {"functions", "function_call", "response_format", "tools", "tool_choice", "top_logprobs"} +) +_PARAMS: Final = TypeAdapter(dict[str, object]) + class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): """ @@ -155,8 +164,12 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): headers: dict, ) -> dict: prompt: Final = transform_prompt(messages) + params: Final = _PARAMS.validate_python(optional_params) + body_only: Final = {key: params.pop(key) for key in _BODY_ONLY_PARAMS & params.keys()} + if body_only: + params["extra_body"] = {**_PARAMS.validate_python(params.get("extra_body") or {}), **body_only} return { "model": model, "prompt": prompt, - **optional_params, + **params, } diff --git a/tests/unit/llms/openai/completion/test_transformation.py b/tests/unit/llms/openai/completion/test_transformation.py new file mode 100644 index 00000000000..4e22e64b88c --- /dev/null +++ b/tests/unit/llms/openai/completion/test_transformation.py @@ -0,0 +1,37 @@ +import json +from typing import Final + +import respx +from httpx import Response + +import litellm + +_COMPLETION: Final = { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "glm", + "choices": [{"index": 0, "text": '{"capital": "Paris"}', "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 5, "completion_tokens": 6, "total_tokens": 11}, +} + + +@respx.mock +def test_text_completion_sends_chat_only_params_in_the_request_body() -> None: + route: Final = respx.post("http://vllm:8000/v1/completions").mock(return_value=Response(200, json=_COMPLETION)) + tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] + + response: Final = litellm.text_completion( + model="hosted_vllm/glm", + api_base="http://vllm:8000/v1", + prompt="The capital of France is", + response_format={"type": "json_object"}, + tools=tools, + tool_choice="auto", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert body["response_format"] == {"type": "json_object"} + assert body["tools"] == tools + assert body["tool_choice"] == "auto" + assert response.choices[0].text == '{"capital": "Paris"}'