From 74e5dc9747a692c6ef54efbbab526928e4155c59 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:17:40 -0700 Subject: [PATCH] fix(proxy): keep provider refusal, logprobs and system_fingerprint in chat completions (#44788) * fix(proxy): keep provider refusal, logprobs and system_fingerprint in chat completions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): handle missing chat completion roles Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): restore throwaway Usage skip in ModelResponse Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): handle explicit fields on custom response dumps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: update refusal expectations for LIT-9235 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): suppress type discipline violations with reasons Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: ruff format proxy utils Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): keep basedpyright counts at merge-base for azure dump and ModelResponse Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): keep an explicit null system_fingerprint passed to ModelResponse Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../convert_dict_to_response.py | 10 +- litellm/llms/azure/azure.py | 14 +- .../llms/databricks/chat/transformation.py | 4 + litellm/llms/openai/common_utils.py | 13 +- litellm/llms/openai/openai.py | 9 +- litellm/proxy/utils.py | 40 ++++- litellm/types/llms/databricks.py | 2 + litellm/types/utils.py | 32 ++-- .../providers/test_dashscope_chat_wire.py | 2 +- .../providers/test_openai_chat_wire.py | 1 - .../providers/test_tencent_chat_wire.py | 1 - .../providers/test_xiaomi_mimo_wire.py | 2 - .../test_chat_completions_basic_azure.py | 3 - .../test_chat_completions_basic_openai.py | 3 - .../test_convert_dict_to_response.py | 6 +- tests/unit/llms/openai/test_openai.py | 44 ++++++ .../test_model_dump_with_preserved_fields.py | 149 +++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 18 files changed, 290 insertions(+), 47 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 7b2860b9f97..6f16e1e3fb4 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -709,7 +709,7 @@ def convert_to_model_response_object( message = Message( content=content, - role=choice["message"]["role"] or "assistant", + role=choice["message"].get("role") or "assistant", function_call=choice["message"].get("function_call", None), tool_calls=tool_calls, audio=choice["message"].get("audio", None), @@ -720,6 +720,8 @@ def convert_to_model_response_object( images=_normalize_images_for_message(choice["message"].get("images", None)), ) finish_reason = choice.get("finish_reason", None) + if "refusal" in choice["message"]: + message.refusal = choice["message"]["refusal"] if finish_reason is None: # gpt-4 vision can return 'finish_reason' or 'finish_details' finish_reason = choice.get("finish_details") or "stop" @@ -728,9 +730,10 @@ def convert_to_model_response_object( ## PROVIDER SPECIFIC FIELDS ## provider_specific_fields = {f: choice[f] for f in choice.keys() - _CHOICES_FIELDS} - logprobs = choice.get("logprobs", None) + provider_sent_logprobs = "logprobs" in choice enhancements = choice.get("enhancements", None) + choice = Choices( finish_reason=finish_reason, index=idx, @@ -739,6 +742,9 @@ def convert_to_model_response_object( enhancements=enhancements, provider_specific_fields=provider_specific_fields, ) + if provider_sent_logprobs and logprobs is None: + choice.logprobs = None + choice_list.append(choice) model_response_object.choices = choice_list diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index d396e1f4867..67b25bf7d12 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -13,6 +13,7 @@ from openai import ( BadRequestError, OpenAI, ) +from pydantic import BaseModel import litellm from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES @@ -41,6 +42,7 @@ from ..base import BaseLLM from ..openai.common_utils import ( build_output_token_limit_response, is_output_token_limit_error, + model_dump_provider_response, ) from .common_utils import ( AzureOpenAIError, @@ -352,12 +354,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): headers, response = self.make_sync_azure_openai_chat_completion_request( azure_client=azure_client, data=data, timeout=timeout ) - if isinstance(response, str): + if not isinstance(response, BaseModel): raise AzureOpenAIError( status_code=500, - message=f"Unexpected string response from Azure: {response[:500]}", + message=f"Unexpected response from Azure: {response!r:.500}", ) - stringified_response: Final = response.model_dump() + stringified_response: Final = model_dump_provider_response(response) ## LOGGING logging_obj.post_call( input=messages, @@ -447,12 +449,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): ) logging_obj.model_call_details["response_headers"] = headers - if isinstance(response, str): + if not isinstance(response, BaseModel): raise AzureOpenAIError( status_code=500, - message=f"Unexpected string response from Azure: {response[:500]}", + message=f"Unexpected response from Azure: {response!r:.500}", ) - stringified_response: Final = response.model_dump() + stringified_response: Final = model_dump_provider_response(response) logging_obj.post_call( input=data["messages"], api_key=api_key, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index dceec658824..94a74d703df 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -697,6 +697,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): model_response.model = f"{_custom_llm_provider}/{_response_model}" model_response.id = completion_response["id"] model_response.created = completion_response["created"] + if "system_fingerprint" in completion_response: + model_response.system_fingerprint = ( # rebind-ok: fills the caller's model_response like id above + completion_response["system_fingerprint"] + ) setattr(model_response, "usage", Usage(**completion_response["usage"])) served_tier: Final = completion_response.get("service_tier") if isinstance(served_tier, str) and served_tier: diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 839ca04e082..4ab91392ad4 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -12,7 +12,7 @@ import time import uuid from collections.abc import AsyncIterator, Iterator, Mapping from contextlib import suppress -from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Optional +from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Optional, Protocol from urllib.parse import urlsplit import httpx @@ -23,6 +23,7 @@ from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta from openai.types.completion_usage import CompletionUsage +from pydantic import BaseModel if TYPE_CHECKING: from aiohttp import ClientSession @@ -51,6 +52,16 @@ _AZURE_OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(Azur _OPENAI_API_HOST: Final[str] = "api.openai.com" +class _ModelDumpable(Protocol): + def model_dump(self) -> dict[str, object]: ... # mutable-ok: pydantic model_dump returns a dict + + +def model_dump_provider_response(response: _ModelDumpable) -> dict[str, object]: # mutable-ok: callers need a dict + if isinstance(response, BaseModel): + return response.model_dump(exclude_unset=True) + return response.model_dump() + + _OPENAI_HTTPX_DEFAULT_TIMEOUT: Final = CompletionTimeout.normalize(openai.DEFAULT_TIMEOUT) _OPENAI_HTTPX_CONNECTION_LIMITS: Final = httpx.Limits( max_connections=openai.DEFAULT_CONNECTION_LIMITS.max_connections, diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 3a12326d433..58eadeadc1e 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -60,6 +60,7 @@ from .common_utils import ( drop_params_from_unprocessable_entity_error, is_openai_backed_api_base, is_output_token_limit_error, + model_dump_provider_response, ) from .workload_identity import resolve_openai_workload_identity_config @@ -852,7 +853,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): ) logging_obj.model_call_details["response_headers"] = headers - stringified_response = provider_config.transform_parsed_response_dict(response.model_dump()) + stringified_response = provider_config.transform_parsed_response_dict( + model_dump_provider_response(response) + ) logging_obj.post_call( input=messages, api_key=api_key, @@ -990,7 +993,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): timeout=timeout, logging_obj=logging_obj, ) - stringified_response = provider_config.transform_parsed_response_dict(response.model_dump()) + stringified_response = provider_config.transform_parsed_response_dict( + model_dump_provider_response(response) + ) logging_obj.post_call( input=data["messages"], api_key=api_key, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 5fa78a44d09..267c87694b9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -111,7 +111,7 @@ except ImportError: raise ImportError("backoff is not installed. Please install it via 'pip install backoff'") from fastapi import HTTPException, status -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError import litellm import litellm.litellm_core_utils @@ -9308,12 +9308,15 @@ def validate_model_access( _PRESERVED_NONE_FIELDS: Final[list[tuple[str, str]]] = [ ("message", "content"), # null when tool_calls present (issue #6677) ("message", "role"), # always required by OpenAI spec + ("message", "refusal"), # explicit null from provider ("delta", "content"), # null in streaming chunks ] +_PRESERVED_NONE_CHOICE_FIELDS: Final = ("logprobs",) # explicit null from provider +_PRESERVED_NONE_RESPONSE_FIELDS: Final = ("system_fingerprint",) # explicit null from provider def model_dump_with_preserved_fields( - obj: Any, + obj: BaseModel, preserve_fields: list[str] | None = None, exclude_unset: bool = True, ) -> dict[str, object]: @@ -9321,7 +9324,7 @@ def model_dump_with_preserved_fields( Serialize a Pydantic model to a dictionary while preserving specific fields even if they are None. - Fields listed in _PRESERVED_NONE_FIELDS are restored after + Explicit null values in the preserved field lists are restored after model_dump(exclude_none=True) strips them. Args: @@ -9332,21 +9335,42 @@ def model_dump_with_preserved_fields( Returns: Dictionary representation with None values excluded except for preserved fields """ - result: Final = obj.model_dump(exclude_none=True, exclude_unset=exclude_unset) + result: Final = cast( # cast-ok: Pydantic model_dump returns serialized model fields + dict[str, object], + obj.model_dump(exclude_none=True, exclude_unset=exclude_unset), + ) - choices: Final = result.get("choices") + for field_name in _PRESERVED_NONE_RESPONSE_FIELDS: + if field_name not in result and field_name in getattr(obj, "model_fields_set", ()): + result[field_name] = getattr(obj, field_name) + + choices: Final = cast( # cast-ok: model_dump serializes choices as a list of dicts + Sequence[dict[str, object]] | None, + result.get("choices"), + ) if not choices: return result - obj_choices: Final = obj.choices + obj_choices: Final = cast( # cast-ok: obj is duck-typed, choices is a list on ModelResponse + Sequence[object], + getattr(obj, "choices", ()), + ) for choice_obj, choice_dict in zip(obj_choices, choices): for sub_object, field_name in _PRESERVED_NONE_FIELDS: - sub_dict = choice_dict.get(sub_object) + sub_dict = cast( # cast-ok: message and delta dump as dicts + dict[str, object] | None, choice_dict.get(sub_object) + ) if sub_dict is None: continue if field_name not in sub_dict: - sub_obj = getattr(choice_obj, sub_object, None) + sub_obj = cast( # cast-ok: getattr on a duck-typed choice + object | None, getattr(choice_obj, sub_object, None) + ) if sub_obj is not None and hasattr(sub_obj, field_name): sub_dict[field_name] = getattr(sub_obj, field_name) + for field_name in _PRESERVED_NONE_CHOICE_FIELDS: + if field_name not in choice_dict and hasattr(choice_obj, field_name): + choice_dict[field_name] = getattr(choice_obj, field_name) + return result diff --git a/litellm/types/llms/databricks.py b/litellm/types/llms/databricks.py index a9c027bd2de..c7023e35777 100644 --- a/litellm/types/llms/databricks.py +++ b/litellm/types/llms/databricks.py @@ -2,6 +2,7 @@ from typing import Any, Literal from pydantic import BaseModel from typing_extensions import ( + NotRequired, ReadOnly, Required, TypedDict, @@ -82,3 +83,4 @@ class DatabricksResponse(TypedDict): model: str choices: list[DatabricksChoice] usage: ChatCompletionUsageBlock + system_fingerprint: NotRequired[ReadOnly[str | None]] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 06af506076a..2d98bf0486f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1416,6 +1416,7 @@ def add_provider_specific_fields(object: BaseModel, provider_specific_fields: di class Message(SafeAttributeModel, OpenAIObject): content: str | None role: Literal["assistant", "user", "system", "tool", "function"] + refusal: str | None = None tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None function_call: FunctionCall | None audio: ChatCompletionAudioResponse | None = None @@ -1493,6 +1494,9 @@ class Message(SafeAttributeModel, OpenAIObject): if hasattr(self, "annotations"): del self.annotations + if "refusal" not in params and hasattr(self, "refusal"): + del self.refusal + if reasoning_content is None: # ensure default response matches OpenAI spec if hasattr(self, "reasoning_content"): @@ -2212,6 +2216,20 @@ class ModelResponseStream(ModelResponseBase): return self.dict() +def _normalize_model_response_usage(usage: Mapping[str, object] | BaseModel) -> Usage: + if isinstance(usage, dict): + return Usage(**usage) + dump: Final = usage.model_dump() if hasattr(usage, "model_dump") else usage.dict() + return Usage(**dump) + + +class _UnsetSystemFingerprint: + __slots__ = () + + +_UNSET_SYSTEM_FINGERPRINT: Final = _UnsetSystemFingerprint() + + class ModelResponse(ModelResponseBase): choices: list[Choices] """The list of completion choices the model generated for the input prompt.""" @@ -2223,7 +2241,7 @@ class ModelResponse(ModelResponseBase): created=None, model=None, object=None, - system_fingerprint=None, + system_fingerprint=_UNSET_SYSTEM_FINGERPRINT, usage=None, stream=None, stream_options=None, @@ -2259,13 +2277,8 @@ class ModelResponse(ModelResponseBase): created = created model = model if usage is not None: - if isinstance(usage, dict): - usage = Usage(**usage) - elif isinstance(usage, BaseModel): - dump = usage.model_dump() if hasattr(usage, "model_dump") else usage.dict() - usage = Usage(**dump) - else: - usage = usage + if isinstance(usage, dict | BaseModel): + usage = _normalize_model_response_usage(usage) elif stream is None or stream is False: usage = None # avoid constructing throwaway Usage; set by convert_to_model_response_object if hidden_params: @@ -2280,11 +2293,12 @@ class ModelResponse(ModelResponseBase): "created": created, "model": model, "object": object, - "system_fingerprint": system_fingerprint, } if usage is not None: init_values["usage"] = usage + if system_fingerprint is not _UNSET_SYSTEM_FINGERPRINT: + init_values["system_fingerprint"] = system_fingerprint super().__init__( **init_values, diff --git a/tests/integration/providers/test_dashscope_chat_wire.py b/tests/integration/providers/test_dashscope_chat_wire.py index a2b3a36d6e3..87933d1eaf9 100644 --- a/tests/integration/providers/test_dashscope_chat_wire.py +++ b/tests/integration/providers/test_dashscope_chat_wire.py @@ -55,7 +55,7 @@ def test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider(gateway: { "finish_reason": "stop", "index": 0, - "message": {"role": "assistant", "content": "27", "provider_specific_fields": {"refusal": None}}, + "message": {"role": "assistant", "content": "27"}, "provider_specific_fields": {}, } ] diff --git a/tests/integration/providers/test_openai_chat_wire.py b/tests/integration/providers/test_openai_chat_wire.py index 8113a37442a..5bb8a1f2c96 100644 --- a/tests/integration/providers/test_openai_chat_wire.py +++ b/tests/integration/providers/test_openai_chat_wire.py @@ -61,7 +61,6 @@ def test_openai_chat_tool_choice_without_tools_is_not_forwarded(gateway: Gateway "message": { "role": "assistant", "content": "One sentence.", - "provider_specific_fields": {"refusal": None}, }, "provider_specific_fields": {}, } diff --git a/tests/integration/providers/test_tencent_chat_wire.py b/tests/integration/providers/test_tencent_chat_wire.py index 84e9eb8ffea..25e5cac0fb6 100644 --- a/tests/integration/providers/test_tencent_chat_wire.py +++ b/tests/integration/providers/test_tencent_chat_wire.py @@ -77,7 +77,6 @@ def test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_reques "role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43.", - "provider_specific_fields": {"refusal": None}, }, "provider_specific_fields": {}, } diff --git a/tests/integration/providers/test_xiaomi_mimo_wire.py b/tests/integration/providers/test_xiaomi_mimo_wire.py index 96b9dc17bdf..2722e501c47 100644 --- a/tests/integration/providers/test_xiaomi_mimo_wire.py +++ b/tests/integration/providers/test_xiaomi_mimo_wire.py @@ -136,7 +136,6 @@ def test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price(gat "role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43.", - "provider_specific_fields": {"refusal": None}, }, "provider_specific_fields": {}, } @@ -250,7 +249,6 @@ def test_xiaomi_mimo_tool_call_is_forwarded_and_returned(gateway: Gateway) -> No "content": None, "reasoning_content": "Need the tool.", "tool_calls": [tool_call], - "provider_specific_fields": {"refusal": None}, }, "provider_specific_fields": {}, } diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py index e19b5223804..b388d303228 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py @@ -24,7 +24,4 @@ from integration.translation.runner import assert_translation ids=lambda case: case.id, ) def test_chat_completions_basic_azure(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: - pytest.skip( - "BUG: LIT-9235 chat completions moves message.refusal into provider_specific_fields and drops null system_fingerprint and logprobs" - ) assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py index f8d46ee28ac..daa54612208 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py @@ -24,7 +24,4 @@ from integration.translation.runner import assert_translation ids=lambda case: case.id, ) def test_chat_completions_basic_openai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: - pytest.skip( - "BUG: LIT-9235 chat completions moves message.refusal into provider_specific_fields and drops null system_fingerprint and logprobs" - ) assert_translation(case, gateway, provider) diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 44d823fa82f..8fdc6c8ba47 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1580,10 +1580,8 @@ def test_convert_to_model_response_object_no_provider_specific_fields_still_work ) assert isinstance(result, ModelResponse) - psf = result.choices[0].message.provider_specific_fields - # refusal is not a Message model field, so it should be in provider_specific_fields - assert psf is not None - assert "refusal" in psf + assert result.choices[0].message.refusal is None + assert result.choices[0].message.provider_specific_fields is None def test_convert_to_model_response_object_with_error_code_only(): """ diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 788269dd043..713b331f8a5 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -38,6 +38,7 @@ from litellm.utils import _invalidate_model_cost_lowercase_map from openai.types.beta.assistant import Assistant from openai.types.beta.assistant_deleted import AssistantDeleted from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +from litellm.proxy.utils import model_dump_with_preserved_fields @pytest.mark.parametrize( @@ -220,6 +221,49 @@ async def test_acompletion_returns_prompt_cache_details_over_injected_transport( assert prompt_tokens_details.cached_tokens == 7 +@respx.mock +def test_completion_defaults_missing_message_role_and_preserves_omitted_fields() -> None: + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-missing-role", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.4", + "choices": [{"index": 0, "message": {"content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + response: Final = litellm.completion( + model="openai/gpt-5.4", + api_base="https://api.openai.com/v1", + api_key="sk-test", + messages=[{"role": "user", "content": "Hi"}], + num_retries=0, + ) + + assert route.called + assert response.choices[0].message.role == "assistant" + assert model_dump_with_preserved_fields(response, exclude_unset=True) == { + "id": "chatcmpl-missing-role", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "hi", "role": "assistant"}, + "provider_specific_fields": {}, + } + ], + "created": 1, + "model": "gpt-5.4", + "object": "chat.completion", + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + @pytest.mark.asyncio @pytest.mark.parametrize( ("status", "expected_error"), diff --git a/tests/unit/proxy/test_model_dump_with_preserved_fields.py b/tests/unit/proxy/test_model_dump_with_preserved_fields.py index 316c1e879cc..31531dccd16 100644 --- a/tests/unit/proxy/test_model_dump_with_preserved_fields.py +++ b/tests/unit/proxy/test_model_dump_with_preserved_fields.py @@ -2,12 +2,20 @@ Regression tests for model_dump_with_preserved_fields. This function serializes ModelResponse / ModelResponseStream objects to dicts -while preserving 3 specific None fields for OpenAI API compatibility: +while preserving specific fields for OpenAI API compatibility: - choices[*].message.content (null when tool_calls present) - choices[*].message.role (always present) - choices[*].delta.content (null in streaming chunks) + - choices[*].message.refusal (explicit null from provider) + - choices[*].logprobs (explicit null from provider) + - system_fingerprint (explicit null from provider) """ +from typing import Final + +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_model_response_object, +) from litellm.proxy.utils import model_dump_with_preserved_fields from litellm.types.utils import ( Choices, @@ -118,8 +126,23 @@ def test_none_fields_stripped_from_message(): assert "audio" not in msg -def test_none_fields_stripped_from_top_level(): - """system_fingerprint=None should be omitted from top-level.""" +def test_unset_system_fingerprint_stripped_from_top_level(): + """A system_fingerprint the provider never sent should be omitted from top-level.""" + response = ModelResponse( + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="Hello", role="assistant"), + ) + ], + ) + result = model_dump_with_preserved_fields(response, exclude_unset=True) + assert "system_fingerprint" not in result + + +def test_explicit_none_system_fingerprint_kept_at_top_level(): + """A provider that sent system_fingerprint: null keeps it through ModelResponse(**payload).""" response = ModelResponse( choices=[ Choices( @@ -131,7 +154,7 @@ def test_none_fields_stripped_from_top_level(): system_fingerprint=None, ) result = model_dump_with_preserved_fields(response, exclude_unset=True) - assert "system_fingerprint" not in result + assert result["system_fingerprint"] is None def test_multiple_choices_independent(): @@ -404,3 +427,121 @@ def test_preserve_fields_param_backward_compat(): assert result_default == result_explicit assert result_default["choices"][0]["message"]["content"] is None assert result_default["choices"][0]["message"]["role"] == "assistant" + + +def test_provider_explicit_null_fields_are_preserved() -> None: + response_object: Final = { + "id": "chatcmpl-provider-null-fields", + "created": 1, + "model": "gpt-5.4", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"role": "assistant", "content": "Hello", "refusal": None}, + "logprobs": None, + } + ], + "system_fingerprint": None, + } + model_response: Final = convert_to_model_response_object( + response_object=response_object, + model_response_object=ModelResponse(), + ) + + assert model_dump_with_preserved_fields(model_response, exclude_unset=True) == { + "id": "chatcmpl-provider-null-fields", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "Hello", "role": "assistant", "refusal": None}, + "provider_specific_fields": {}, + "logprobs": None, + } + ], + "created": 1, + "model": "gpt-5.4", + "object": "chat.completion", + "system_fingerprint": None, + } + + +def test_provider_omitted_null_fields_stay_absent() -> None: + response_object: Final = { + "id": "chatcmpl-provider-omitted-fields", + "created": 2, + "model": "gpt-5.4", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + } + ], + } + model_response: Final = convert_to_model_response_object( + response_object=response_object, + model_response_object=ModelResponse(), + ) + + assert model_dump_with_preserved_fields(model_response, exclude_unset=True) == { + "id": "chatcmpl-provider-omitted-fields", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "Hello", "role": "assistant"}, + "provider_specific_fields": {}, + } + ], + "created": 2, + "model": "gpt-5.4", + "object": "chat.completion", + } + + +def test_provider_refusal_string_is_preserved_on_message() -> None: + response_object: Final = { + "id": "chatcmpl-provider-refusal", + "created": 3, + "model": "gpt-5.4", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "I can't help with that.", + "refusal": "I can't help with that.", + }, + } + ], + } + model_response: Final = convert_to_model_response_object( + response_object=response_object, + model_response_object=ModelResponse(), + ) + + assert model_dump_with_preserved_fields(model_response, exclude_unset=True) == { + "id": "chatcmpl-provider-refusal", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "I can't help with that.", + "role": "assistant", + "refusal": "I can't help with that.", + }, + "provider_specific_fields": {}, + } + ], + "created": 3, + "model": "gpt-5.4", + "object": "chat.completion", + } + assert model_response.choices[0].message.refusal == "I can't help with that." diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f598ad012db..f44c425c2c0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37616,6 +37616,8 @@ export interface components { reasoning_content?: string | null; /** Reasoning Items */ reasoning_items?: components["schemas"]["ChatCompletionReasoningItem"][] | null; + /** Refusal */ + refusal?: string | null; /** * Role * @enum {string}