Merge 42fb38ee6f into 3afe779125

litellm/llms/openai/completion/transformation.py

@@ -5,6 +5,8 @@ Support for gpt model family
 from collections.abc import Mapping
 from typing import Final
 
+from pydantic import TypeAdapter
+
 from litellm.litellm_core_utils.hidden_params import set_hidden_param
 from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig
 from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
@@ -13,6 +15,13 @@ from litellm.types.utils import Choices, Message, ModelResponse, TextCompletionR
 from ..chat.gpt_transformation import OpenAIGPTConfig
 from .utils import transform_prompt
 
+# Supported params the SDK's ``completions.create()`` has no keyword for. They go in ``extra_body``
+# so backends that accept them (e.g. vLLM) receive them, instead of the SDK raising a TypeError.
+_BODY_ONLY_PARAMS: Final = frozenset(
+    {"functions", "function_call", "response_format", "tools", "tool_choice", "top_logprobs"}
+)
+_PARAMS: Final = TypeAdapter(dict[str, object])
+
 
 class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
     """
@@ -155,8 +164,12 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
         headers: dict,
     ) -> dict:
         prompt: Final = transform_prompt(messages)
+        params: Final = _PARAMS.validate_python(optional_params)
+        body_only: Final = {key: params.pop(key) for key in _BODY_ONLY_PARAMS & params.keys()}
+        if body_only:
+            params["extra_body"] = {**_PARAMS.validate_python(params.get("extra_body") or {}), **body_only}
         return {
             "model": model,
             "prompt": prompt,
-            **optional_params,
+            **params,
         }

tests/unit/llms/openai/completion/test_transformation.py

@@ -0,0 +1,37 @@
+import json
+from typing import Final
+
+import respx
+from httpx import Response
+
+import litellm
+
+_COMPLETION: Final = {
+    "id": "cmpl-1",
+    "object": "text_completion",
+    "created": 1,
+    "model": "glm",
+    "choices": [{"index": 0, "text": '{"capital": "Paris"}', "finish_reason": "stop", "logprobs": None}],
+    "usage": {"prompt_tokens": 5, "completion_tokens": 6, "total_tokens": 11},
+}
+
+
+@respx.mock
+def test_text_completion_sends_chat_only_params_in_the_request_body() -> None:
+    route: Final = respx.post("http://vllm:8000/v1/completions").mock(return_value=Response(200, json=_COMPLETION))
+    tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
+
+    response: Final = litellm.text_completion(
+        model="hosted_vllm/glm",
+        api_base="http://vllm:8000/v1",
+        prompt="The capital of France is",
+        response_format={"type": "json_object"},
+        tools=tools,
+        tool_choice="auto",
+    )
+
+    body: Final = json.loads(route.calls.last.request.content)
+    assert body["response_format"] == {"type": "json_object"}
+    assert body["tools"] == tools
+    assert body["tool_choice"] == "auto"
+    assert response.choices[0].text == '{"capital": "Paris"}'