mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(responses): send thought signature fields only with short ids and drop malformed routing kwargs whole
This commit is contained in:
mateo-berri
parent
5b3f7ab64c
commit
3f6b36e8bb
9 changed files with 256 additions and 21 deletions
|
|
@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin
|
|||
from concurrent import futures
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
|
||||
|
|
@ -381,18 +382,27 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
|
|||
return assert_never(control)
|
||||
|
||||
|
||||
def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> GenericLiteLLMParams | None:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SupplementalProviderParams:
|
||||
params: GenericLiteLLMParams | None
|
||||
dropped_keys: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams:
|
||||
supplemental: Final = {key: kwargs[key] for key in OPTIONAL_KWARGS_KEYS if key in kwargs}
|
||||
if not supplemental:
|
||||
return None
|
||||
return _SupplementalProviderParams(params=None)
|
||||
try:
|
||||
return GenericLiteLLMParams.model_validate(supplemental)
|
||||
return _SupplementalProviderParams(params=GenericLiteLLMParams.model_validate(supplemental))
|
||||
except ValidationError as error:
|
||||
invalid: Final = tuple(detail for detail in error.errors() if detail["loc"])
|
||||
invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid)
|
||||
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
|
||||
return GenericLiteLLMParams.model_validate(
|
||||
{key: value for key, value in supplemental.items() if key not in invalid_keys}
|
||||
return _SupplementalProviderParams(
|
||||
params=GenericLiteLLMParams.model_validate(
|
||||
{key: value for key, value in supplemental.items() if key not in invalid_keys}
|
||||
),
|
||||
dropped_keys=invalid_keys,
|
||||
)
|
||||
raise litellm.BadRequestError(
|
||||
message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid),
|
||||
|
|
@ -701,7 +711,7 @@ async def acompletion(
|
|||
"str | None", request_kwargs.get("api_base")
|
||||
)
|
||||
or base_url,
|
||||
litellm_params=_supplemental_provider_params(request_kwargs, model),
|
||||
litellm_params=_supplemental_provider_params(request_kwargs, model).params,
|
||||
)
|
||||
|
||||
fallbacks = fallbacks or litellm.model_fallbacks
|
||||
|
|
@ -5446,6 +5456,9 @@ def completion(
|
|||
### VALIDATE Request ###
|
||||
if model is None:
|
||||
raise ValueError("model param not passed in.")
|
||||
supplemental_params: Final = _supplemental_provider_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), model)
|
||||
for dropped_key in supplemental_params.dropped_keys:
|
||||
kwargs.pop(dropped_key)
|
||||
# validate messages
|
||||
messages = validate_and_fix_openai_messages(messages=messages)
|
||||
tools = validate_and_fix_openai_tools(tools=tools)
|
||||
|
|
@ -5682,10 +5695,7 @@ def completion(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
litellm_params=_supplemental_provider_params(
|
||||
routing_kwargs,
|
||||
cast(str, model), # cast-ok: model is the alias-resolved request model name
|
||||
),
|
||||
litellm_params=supplemental_params.params,
|
||||
)
|
||||
|
||||
## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import Final
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.responses.additional_tools import hoist_additional_tools
|
||||
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
|
||||
LiteLLMCompletionStreamingIterator,
|
||||
|
|
@ -99,6 +100,9 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=bridged_request,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
||||
|
|
@ -150,5 +154,8 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=responses_api_request,
|
||||
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ class ResponsesSessionHandler:
|
|||
previous_response_id: str,
|
||||
item_cache_control: bool = False,
|
||||
client_response_ids: tuple[str, ...] = (),
|
||||
tool_call_provider_fields: bool = False,
|
||||
) -> ChatCompletionSession:
|
||||
"""
|
||||
Return the chat completion message history for a previous response id
|
||||
|
|
@ -125,6 +126,7 @@ class ResponsesSessionHandler:
|
|||
proxy_server_request_dict=proxy_server_request_dict,
|
||||
chat_completion_message_history=chat_completion_message_history,
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -150,6 +152,7 @@ class ResponsesSessionHandler:
|
|||
| Message
|
||||
],
|
||||
item_cache_control: bool = False,
|
||||
tool_call_provider_fields: bool = False,
|
||||
):
|
||||
"""
|
||||
Extend the chat completion message history with the spend log payload
|
||||
|
|
@ -182,6 +185,7 @@ class ResponsesSessionHandler:
|
|||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -196,6 +200,7 @@ class ResponsesSessionHandler:
|
|||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
)
|
||||
)
|
||||
elif isinstance(_response_output, dict) and _response_output and _response_output != {}:
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
responses_api_request: ResponsesAPIOptionalRequestParams,
|
||||
custom_llm_provider: str | None = None,
|
||||
litellm_metadata: dict | None = None,
|
||||
tool_call_provider_fields: bool = False,
|
||||
):
|
||||
self.model: str = model
|
||||
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper
|
||||
|
|
@ -134,6 +135,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request
|
||||
self.custom_llm_provider: str | None = custom_llm_provider
|
||||
self.litellm_metadata: dict | None = litellm_metadata or {}
|
||||
self.tool_call_provider_fields: Final = tool_call_provider_fields
|
||||
# Store lightweight dict snapshots for stream_chunk_builder to reduce
|
||||
# repeated Pydantic attribute access in end-of-stream assembly.
|
||||
self.collected_chat_completion_chunks: list[dict[str, object]] = []
|
||||
|
|
@ -488,7 +490,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=output_index,
|
||||
sequence_number=self._sequence_number,
|
||||
item=BaseLiteLLMOpenAIResponseObject(**item_kwargs, **_tool_call_provider_specific_fields(tc)),
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**item_kwargs,
|
||||
**(_tool_call_provider_specific_fields(tc) if self.tool_call_provider_fields else {}),
|
||||
),
|
||||
)
|
||||
self._pending_tool_events.append(item_done_event)
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ from litellm.litellm_core_utils.hidden_params import (
|
|||
set_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
|
||||
|
|
@ -327,17 +328,32 @@ def _dropped_cache_control(_: object) -> None:
|
|||
return None
|
||||
|
||||
|
||||
def _without_cache_control_key(item: object) -> object:
|
||||
def _without_keys(item: object, keys: frozenset[str]) -> object:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"}
|
||||
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key not in keys}
|
||||
|
||||
|
||||
def _without_item_keys(input: str | ResponseInputParam, keys: frozenset[str]) -> str | ResponseInputParam:
|
||||
if not keys or not isinstance(input, list):
|
||||
return input
|
||||
return cast( # cast-ok: the same input items, each minus the given keys
|
||||
ResponseInputParam, [_without_keys(item, keys) for item in input]
|
||||
)
|
||||
|
||||
|
||||
def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if not isinstance(input, list):
|
||||
return input
|
||||
return cast( # cast-ok: the same input items, each minus its own cache_control key
|
||||
ResponseInputParam, [_without_cache_control_key(item) for item in input]
|
||||
return _without_item_keys(input, frozenset(("cache_control",)))
|
||||
|
||||
|
||||
def _dropped_item_keys(item_cache_control: bool, tool_call_provider_fields: bool) -> frozenset[str]:
|
||||
return frozenset(
|
||||
key
|
||||
for key, kept in (
|
||||
("cache_control", item_cache_control),
|
||||
("provider_specific_fields", tool_call_provider_fields),
|
||||
)
|
||||
if not kept
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -879,6 +895,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
responses_api_request=responses_api_request,
|
||||
replay_reasoning=True,
|
||||
item_cache_control=keeps_item_marks,
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
|
||||
)
|
||||
|
||||
litellm_completion_request: dict = {
|
||||
|
|
@ -940,6 +957,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
responses_api_request: ResponsesAPIOptionalRequestParams | dict,
|
||||
replay_reasoning: bool = False,
|
||||
item_cache_control: bool = False,
|
||||
tool_call_provider_fields: bool = False,
|
||||
) -> list[
|
||||
AllMessageValues
|
||||
| GenericChatCompletionMessage
|
||||
|
|
@ -955,7 +973,9 @@ class LiteLLMCompletionResponsesConfig:
|
|||
assistant ``reasoning_content`` and signed ``thinking_blocks`` so the
|
||||
provider gets its own chain-of-thought back instead of reading it as
|
||||
visible text. ``item_cache_control`` belongs to the same callers, when
|
||||
the target keeps an input item's ``cache_control`` on the message.
|
||||
the target keeps an input item's ``cache_control`` on the message, and
|
||||
``tool_call_provider_fields`` when the target reads a function call's
|
||||
thought signature from ``provider_specific_fields`` instead of its id.
|
||||
|
||||
Callers that only inspect the messages (token counting, rate limiting,
|
||||
guardrail scanning) leave both off, because they need every piece of text
|
||||
|
|
@ -978,7 +998,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
|
||||
messages.extend(
|
||||
LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
|
||||
input=input if item_cache_control else without_item_cache_control(input),
|
||||
input=_without_item_keys(input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)),
|
||||
replay_reasoning=replay_reasoning,
|
||||
)
|
||||
)
|
||||
|
|
@ -1021,6 +1041,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id,
|
||||
item_cache_control=keeps_item_marks,
|
||||
client_response_ids=previous_response_client_ids,
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
|
||||
)
|
||||
chat_completion_session: Final = (
|
||||
await lookup(previous_response_id)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import json
|
|||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -529,9 +530,10 @@ async def test_messages_tool_use_keeps_thought_signature_kept_out_of_its_id(
|
|||
|
||||
|
||||
RESPONSES_WEATHER_TOOL: Final = {"type": "function", **WEATHER_TOOL["function"]}
|
||||
SHORT_IDS: Final = MappingProxyType({"thought_signature_in_tool_call_id": False})
|
||||
|
||||
|
||||
def _responses_function_call_items(stream: bool) -> list[object]:
|
||||
def _responses_function_call_items(stream: bool, request_params: Mapping[str, object] = SHORT_IDS) -> list[object]:
|
||||
with respx.mock:
|
||||
respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_response(None, stream))
|
||||
response: Final = litellm.responses(
|
||||
|
|
@ -540,7 +542,7 @@ def _responses_function_call_items(stream: bool) -> list[object]:
|
|||
tools=[RESPONSES_WEATHER_TOOL],
|
||||
stream=stream,
|
||||
api_key="test-gemini-key",
|
||||
thought_signature_in_tool_call_id=False,
|
||||
**request_params,
|
||||
)
|
||||
items: Final = (
|
||||
[event.item for event in response if event.type == "response.output_item.done"]
|
||||
|
|
@ -559,6 +561,60 @@ def test_responses_function_call_keeps_thought_signature_kept_out_of_its_id(stre
|
|||
assert function_calls[0].provider_specific_fields["thought_signature"] == LONG_SIGNATURE
|
||||
|
||||
|
||||
def test_streamed_function_call_carries_its_thought_signature_only_in_its_id_by_default():
|
||||
function_calls: Final = _responses_function_call_items(True, {})
|
||||
|
||||
assert len(function_calls) == 1
|
||||
assert get_thought_signature_from_tool({"id": function_calls[0].call_id}) == LONG_SIGNATURE
|
||||
assert getattr(function_calls[0], "provider_specific_fields", None) is None
|
||||
|
||||
|
||||
def _deepseek_answer() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "deepseek-v4-flash",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Sunny"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_params", [{}, SHORT_IDS])
|
||||
def test_responses_function_call_provider_fields_reach_the_provider_only_with_short_ids(
|
||||
request_params: Mapping[str, object],
|
||||
):
|
||||
replayed_item: Final = {
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city": "Paris"}',
|
||||
"provider_specific_fields": {"thought_signature": LONG_SIGNATURE},
|
||||
}
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.post(url__regex=r"https://api\.deepseek\.com/.*").mock(return_value=_deepseek_answer())
|
||||
litellm.responses(
|
||||
model="deepseek/deepseek-v4-flash",
|
||||
input=[
|
||||
{"role": "user", "content": USER_TURN["content"]},
|
||||
replayed_item,
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": '{"temp_c": 21}'},
|
||||
],
|
||||
tools=[RESPONSES_WEATHER_TOOL],
|
||||
api_key="test-deepseek-key",
|
||||
**request_params,
|
||||
)
|
||||
sent_body: Final = json.loads(route.calls.last.request.content)
|
||||
|
||||
sent_tool_calls: Final = list(chain.from_iterable(message.get("tool_calls") or () for message in sent_body["messages"]))
|
||||
assert len(sent_tool_calls) == 1
|
||||
assert ("provider_specific_fields" in sent_tool_calls[0]) is bool(request_params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_keeps_provider_fields", [True, False])
|
||||
def test_responses_function_call_replays_its_thought_signature_to_gemini(client_keeps_provider_fields: bool):
|
||||
function_call: Final = _responses_function_call_items(False)[0]
|
||||
|
|
|
|||
|
|
@ -4318,6 +4318,7 @@ class TestEnsureOutputItemContentPartAdded:
|
|||
iterator._queued_web_search_call_ids = set()
|
||||
iterator._tool_search_requested = False
|
||||
iterator._tool_search_call_ids = set()
|
||||
iterator.tool_call_provider_fields = False
|
||||
return iterator
|
||||
|
||||
def _make_text_chunk(self):
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Sequence
|
||||
from itertools import chain
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
import httpx
|
||||
import respx
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
|
@ -955,3 +958,69 @@ async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_
|
|||
|
||||
assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"]
|
||||
assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"]
|
||||
|
||||
|
||||
GEMINI_URL_PATTERN: Final = r"https://generativelanguage\.googleapis\.com/v1[a-z]*/models/gemini-3\.5-flash:.*"
|
||||
LONG_SIGNATURE: Final = "Ab+/" * 200
|
||||
|
||||
|
||||
def _gemini_answer() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"candidates": [
|
||||
{"content": {"role": "model", "parts": [{"text": "Sunny"}]}, "finishReason": "STOP", "index": 0}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _stored_function_call_turn(response_id: str) -> dict:
|
||||
return {
|
||||
"request_id": response_id,
|
||||
"call_type": "aresponses",
|
||||
"session_id": "gemini-session",
|
||||
"proxy_server_request": {"input": "What is the weather in Paris?", "model": "gemini-short-ids"},
|
||||
"response": {
|
||||
"id": response_id,
|
||||
"object": "response",
|
||||
"output": [
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city": "Paris"}',
|
||||
"status": "completed",
|
||||
"provider_specific_fields": {"thought_signature": LONG_SIGNATURE},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("short_ids", [False, True])
|
||||
async def test_previous_response_replays_a_thought_signature_kept_out_of_the_id_only_with_short_ids(
|
||||
short_ids: bool, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id(
|
||||
custom_llm_provider="gemini", model_id=None, response_id="chatcmpl-gemini-turn-1"
|
||||
)
|
||||
stored: Final = _StoredSpendLogs((_stored_function_call_turn(response_id),))
|
||||
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored)), respx.mock:
|
||||
route: Final = respx.post(url__regex=GEMINI_URL_PATTERN).mock(return_value=_gemini_answer())
|
||||
await litellm.aresponses(
|
||||
model="gemini/gemini-3.5-flash",
|
||||
input=[{"type": "function_call_output", "call_id": "call_1", "output": '{"temp_c": 21}'}],
|
||||
previous_response_id=response_id,
|
||||
api_key="test-gemini-key",
|
||||
**({"thought_signature_in_tool_call_id": False} if short_ids else {}),
|
||||
)
|
||||
sent_body: Final = json.loads(route.calls.last.request.content)
|
||||
|
||||
model_turns: Final = [turn for turn in sent_body["contents"] if turn["role"] == "model"]
|
||||
model_parts: Final = list(chain.from_iterable(turn["parts"] for turn in model_turns))
|
||||
assert len(model_parts) == 1
|
||||
assert (model_parts[0]["thoughtSignature"] == LONG_SIGNATURE) is short_ids
|
||||
|
|
|
|||
|
|
@ -9810,3 +9810,64 @@ async def test_converse_fallback_prices_the_current_attempt(
|
|||
else:
|
||||
client.close()
|
||||
configuration.reset_rust_configuration()
|
||||
|
||||
|
||||
def _openai_chat_answer() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _openai_completion(use_async: bool, **request_params: object) -> litellm.ModelResponse:
|
||||
request: Final = {
|
||||
"model": "openai/gpt-5.6",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"api_key": "fake-key",
|
||||
**request_params,
|
||||
}
|
||||
return await litellm.acompletion(**request) if use_async else litellm.completion(**request)
|
||||
|
||||
|
||||
class _PreCallParamsRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.litellm_params: tuple[object, ...] = ()
|
||||
|
||||
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
|
||||
self.litellm_params = (*self.litellm_params, kwargs.get("litellm_params"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
async def test_malformed_routing_kwarg_answers_400_before_reaching_the_provider(use_async: bool):
|
||||
with respx.mock(assert_all_called=False) as mock:
|
||||
route: Final = mock.post("https://api.openai.com/v1/chat/completions").mock(return_value=_openai_chat_answer())
|
||||
with pytest.raises(litellm.BadRequestError, match="tpm") as raised:
|
||||
await _openai_completion(use_async, tpm="abc")
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
async def test_malformed_routing_kwarg_is_dropped_from_the_whole_request_under_drop_params(
|
||||
use_async: bool, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
recorder: Final = _PreCallParamsRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
with respx.mock:
|
||||
respx.post("https://api.openai.com/v1/chat/completions").mock(return_value=_openai_chat_answer())
|
||||
response: Final = await _openai_completion(use_async, tpm="abc", drop_params=True)
|
||||
|
||||
assert response.choices[0].message.content == "hi"
|
||||
assert len(recorder.litellm_params) == 1
|
||||
assert isinstance(recorder.litellm_params[0], Mapping)
|
||||
assert recorder.litellm_params[0].get("tpm") is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue