From 7836a5401806f9db6f2aa9d8f1a349017b5df415 Mon Sep 17 00:00:00 2001 From: Prasanna Jain Date: Thu, 22 Oct 2026 20:14:06 -0400 Subject: [PATCH 1/3] fix(text-completion): send chat-only params in the request body instead of as SDK kwargs OpenAITextCompletionConfig lists response_format, tools, tool_choice, functions, function_call and top_logprobs as supported, then spreads them as keyword arguments into the OpenAI SDK's completions.create(), which has no such keywords. The SDK raised a TypeError and the proxy answered 500 before any request reached the backend. Pass them via extra_body so backends that accept them (e.g. vLLM) receive them and others answer with their own 4xx. Fixes #47951 --- .../llms/openai/completion/transformation.py | 12 +++++- .../openai/completion/test_transformation.py | 37 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/unit/llms/openai/completion/test_transformation.py diff --git a/litellm/llms/openai/completion/transformation.py b/litellm/llms/openai/completion/transformation.py index f7d2ce68082..a6c9308b0e5 100644 --- a/litellm/llms/openai/completion/transformation.py +++ b/litellm/llms/openai/completion/transformation.py @@ -13,6 +13,12 @@ from litellm.types.utils import Choices, Message, ModelResponse, TextCompletionR from ..chat.gpt_transformation import OpenAIGPTConfig from .utils import transform_prompt +# Supported params the SDK's ``completions.create()`` has no keyword for. They go in ``extra_body`` +# so backends that accept them (e.g. vLLM) receive them, instead of the SDK raising a TypeError. +_BODY_ONLY_PARAMS: Final = frozenset( + {"functions", "function_call", "response_format", "tools", "tool_choice", "top_logprobs"} +) + class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): """ @@ -155,8 +161,12 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): headers: dict, ) -> dict: prompt: Final = transform_prompt(messages) + body_only: Final = {k: v for k, v in optional_params.items() if k in _BODY_ONLY_PARAMS} + params: Final = {k: v for k, v in optional_params.items() if k not in _BODY_ONLY_PARAMS} + if body_only: + params["extra_body"] = {**(params.get("extra_body") or {}), **body_only} return { "model": model, "prompt": prompt, - **optional_params, + **params, } diff --git a/tests/unit/llms/openai/completion/test_transformation.py b/tests/unit/llms/openai/completion/test_transformation.py new file mode 100644 index 00000000000..4e22e64b88c --- /dev/null +++ b/tests/unit/llms/openai/completion/test_transformation.py @@ -0,0 +1,37 @@ +import json +from typing import Final + +import respx +from httpx import Response + +import litellm + +_COMPLETION: Final = { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "glm", + "choices": [{"index": 0, "text": '{"capital": "Paris"}', "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 5, "completion_tokens": 6, "total_tokens": 11}, +} + + +@respx.mock +def test_text_completion_sends_chat_only_params_in_the_request_body() -> None: + route: Final = respx.post("http://vllm:8000/v1/completions").mock(return_value=Response(200, json=_COMPLETION)) + tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] + + response: Final = litellm.text_completion( + model="hosted_vllm/glm", + api_base="http://vllm:8000/v1", + prompt="The capital of France is", + response_format={"type": "json_object"}, + tools=tools, + tool_choice="auto", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert body["response_format"] == {"type": "json_object"} + assert body["tools"] == tools + assert body["tool_choice"] == "auto" + assert response.choices[0].text == '{"capital": "Paris"}' From a4c5dfb0556542d25013195bb7b3439d387c5986 Mon Sep 17 00:00:00 2001 From: Prasanna Jain Date: Thu, 22 Oct 2026 20:30:21 -0400 Subject: [PATCH 2/3] fix(text-completion): keep body-only param split fully typed for the basedpyright gate --- litellm/llms/openai/completion/transformation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/llms/openai/completion/transformation.py b/litellm/llms/openai/completion/transformation.py index a6c9308b0e5..eb32853be31 100644 --- a/litellm/llms/openai/completion/transformation.py +++ b/litellm/llms/openai/completion/transformation.py @@ -3,7 +3,7 @@ Support for gpt model family """ from collections.abc import Mapping -from typing import Final +from typing import Final, cast from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig @@ -161,10 +161,10 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): headers: dict, ) -> dict: prompt: Final = transform_prompt(messages) - body_only: Final = {k: v for k, v in optional_params.items() if k in _BODY_ONLY_PARAMS} - params: Final = {k: v for k, v in optional_params.items() if k not in _BODY_ONLY_PARAMS} + params: Final = cast(dict[str, object], optional_params).copy() + body_only: Final = {key: params.pop(key) for key in _BODY_ONLY_PARAMS & params.keys()} if body_only: - params["extra_body"] = {**(params.get("extra_body") or {}), **body_only} + params["extra_body"] = {**cast(Mapping[str, object], params.get("extra_body") or {}), **body_only} return { "model": model, "prompt": prompt, From 42fb38ee6f23158a50882cb9177f4bf7f9820a6f Mon Sep 17 00:00:00 2001 From: Prasanna Jain Date: Thu, 22 Oct 2026 20:45:09 -0400 Subject: [PATCH 3/3] fix(text-completion): validate params with a TypeAdapter instead of typing.cast --- litellm/llms/openai/completion/transformation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai/completion/transformation.py b/litellm/llms/openai/completion/transformation.py index eb32853be31..28c8e1372fb 100644 --- a/litellm/llms/openai/completion/transformation.py +++ b/litellm/llms/openai/completion/transformation.py @@ -3,7 +3,9 @@ Support for gpt model family """ from collections.abc import Mapping -from typing import Final, cast +from typing import Final + +from pydantic import TypeAdapter from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig @@ -18,6 +20,7 @@ from .utils import transform_prompt _BODY_ONLY_PARAMS: Final = frozenset( {"functions", "function_call", "response_format", "tools", "tool_choice", "top_logprobs"} ) +_PARAMS: Final = TypeAdapter(dict[str, object]) class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): @@ -161,10 +164,10 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): headers: dict, ) -> dict: prompt: Final = transform_prompt(messages) - params: Final = cast(dict[str, object], optional_params).copy() + params: Final = _PARAMS.validate_python(optional_params) body_only: Final = {key: params.pop(key) for key in _BODY_ONLY_PARAMS & params.keys()} if body_only: - params["extra_body"] = {**cast(Mapping[str, object], params.get("extra_body") or {}), **body_only} + params["extra_body"] = {**_PARAMS.validate_python(params.get("extra_body") or {}), **body_only} return { "model": model, "prompt": prompt,