mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(responses): preserve text verbosity when bridging to chat completions (#49049)
* test(responses): cover text verbosity on the chat completions bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): send text.verbosity as chat verbosity on the chat completions bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(responses): type the bridged text verbosity lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): cover text verbosity on the client tool search follow-up bridge turn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): cover text verbosity validation on the codex tool search follow-up turn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): send text.verbosity on the client tool search follow-up bridge turn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): cover text.verbosity on previous_response_id client tool_search follow-up Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ryan-crabbe-berri
GitHub
parent
61efbd071e
commit
5950af8fb5
4 changed files with 763 additions and 9 deletions
|
|
@ -33,6 +33,14 @@ from litellm.types.utils import ModelResponse
|
|||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match responses_api_request.get("text"):
|
||||
case {"verbosity": str() as verbosity}:
|
||||
return {"verbosity": verbosity}
|
||||
case _:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BridgedRequest:
|
||||
input: str | ResponseInputParam
|
||||
|
|
@ -95,8 +103,8 @@ class LiteLLMCompletionTransformationHandler:
|
|||
bridged: Final = _bridged_request(hoisted, responses_api_request, ())
|
||||
bridged_input: Final = bridged.input
|
||||
bridged_request: Final = bridged.responses_api_request
|
||||
litellm_completion_request: Final[dict] = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
litellm_completion_request: Final[dict] = {
|
||||
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input=bridged_input,
|
||||
responses_api_request=bridged_request,
|
||||
|
|
@ -104,8 +112,9 @@ class LiteLLMCompletionTransformationHandler:
|
|||
stream=stream,
|
||||
extra_headers=extra_headers,
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
),
|
||||
**_chat_verbosity(bridged_request),
|
||||
}
|
||||
|
||||
if _is_async:
|
||||
return self.async_response_api_handler(
|
||||
|
|
@ -169,8 +178,8 @@ class LiteLLMCompletionTransformationHandler:
|
|||
return await self.async_response_api_handler(
|
||||
previous_response_client_ids,
|
||||
stored_turns,
|
||||
litellm_completion_request=(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
litellm_completion_request={
|
||||
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input=bridged.input,
|
||||
responses_api_request=bridged.responses_api_request,
|
||||
|
|
@ -178,8 +187,9 @@ class LiteLLMCompletionTransformationHandler:
|
|||
stream=stream,
|
||||
extra_headers=extra_headers,
|
||||
**request_kwargs,
|
||||
)
|
||||
),
|
||||
),
|
||||
**_chat_verbosity(bridged.responses_api_request),
|
||||
},
|
||||
request_input=bridged.input,
|
||||
responses_api_request=bridged.responses_api_request,
|
||||
**request_kwargs,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,413 @@
|
|||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support import responses_vendor as rv
|
||||
from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment
|
||||
from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply
|
||||
from integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(120)
|
||||
|
||||
_GPT: Final = "gpt-5.6"
|
||||
_DATABRICKS: Final = "databricks-meta-llama-4-maverick"
|
||||
_PROVIDER_KEY: Final = "synthetic-provider-key"
|
||||
_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True}
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}, "celsius": {"type": "number"}},
|
||||
"required": ["city", "celsius"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
_TEXT_FORMAT: Final[dict[str, JsonValue]] = {
|
||||
"type": "json_schema",
|
||||
"name": "weather",
|
||||
"schema": _WEATHER_SCHEMA,
|
||||
"strict": True,
|
||||
}
|
||||
_RESPONSE_FORMAT: Final[dict[str, JsonValue]] = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "weather", "schema": _WEATHER_SCHEMA, "strict": True},
|
||||
}
|
||||
_STREAM_OPTIONS: Final[dict[str, JsonValue]] = {"stream_options": {"include_usage": True}}
|
||||
|
||||
Verbosity = Literal["low", "high"]
|
||||
|
||||
|
||||
def _prompt(marker: str) -> str:
|
||||
return f"verbosity probe marker-{marker}"
|
||||
|
||||
|
||||
def _answer(marker: str) -> str:
|
||||
return f"scripted answer marker-{marker}"
|
||||
|
||||
|
||||
def _marker_of(request: Request) -> str:
|
||||
body: Final = request.body.decode()
|
||||
start: Final = body.rfind("marker-")
|
||||
return body[start + len("marker-") : start + len("marker-") + 32] if start >= 0 else "health"
|
||||
|
||||
|
||||
def _vendor(request: Request) -> Reply:
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body or b"{}")
|
||||
marker: Final = _marker_of(request)
|
||||
stream: Final = body.get("stream") is True
|
||||
if request.target.endswith("/responses"):
|
||||
return responses_reply(f"resp_{marker}", _GPT, _answer(marker), stream=stream)
|
||||
return chat_reply(f"chatcmpl-{marker}", str(body.get("model")), _answer(marker), stream=stream)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _upstream() -> Iterator[Wire]:
|
||||
with wire_server(answering_model_discovery(_vendor)) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
def _bridged_gpt(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
|
||||
return scenario.model(
|
||||
model=f"openai/{_GPT}",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key=_PROVIDER_KEY,
|
||||
use_chat_completions_api=True,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
|
||||
def _native_gpt(scenario: Scenario, wire: Wire) -> str:
|
||||
return scenario.model(model=f"openai/{_GPT}", api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY)
|
||||
|
||||
|
||||
def _databricks(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
|
||||
return scenario.model(
|
||||
model=f"databricks/{_DATABRICKS}",
|
||||
api_base=f"{wire.url}/serving-endpoints",
|
||||
api_key=_PROVIDER_KEY,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
|
||||
def _responses(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> httpx.Response:
|
||||
return gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": _prompt(marker), "cache": _NO_CACHE, **fields}
|
||||
)
|
||||
|
||||
|
||||
def _streamed_events(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> list[dict[str, JsonValue]]:
|
||||
with gateway.client.stream(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
json={"model": model, "input": _prompt(marker), "cache": _NO_CACHE, "stream": True, **fields},
|
||||
headers={"Authorization": f"Bearer {gateway.key}"},
|
||||
) as response:
|
||||
assert response.status_code == 200, response.read()
|
||||
return [
|
||||
_JSON_OBJECT.validate_json(line.removeprefix("data: "))
|
||||
for line in response.iter_lines()
|
||||
if line.startswith("data: {")
|
||||
]
|
||||
|
||||
|
||||
def _output_text(response: Mapping[str, JsonValue]) -> str:
|
||||
output: Final = response["output"]
|
||||
assert isinstance(output, list), response
|
||||
return "".join(
|
||||
str(part["text"])
|
||||
for item in output
|
||||
if isinstance(item, dict) and item.get("type") == "message"
|
||||
for part in item["content"]
|
||||
if isinstance(part, dict) and part.get("type") == "output_text"
|
||||
)
|
||||
|
||||
|
||||
def _answered(response: httpx.Response, marker: str) -> None:
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = _JSON_OBJECT.validate_json(response.text)
|
||||
assert body["status"] == "completed" and _output_text(body) == _answer(marker), body
|
||||
|
||||
|
||||
def _streamed_answer(events: list[dict[str, JsonValue]], marker: str) -> None:
|
||||
completed: Final = [event for event in events if event["type"] == "response.completed"]
|
||||
assert len(completed) == 1, [event["type"] for event in events]
|
||||
final: Final = completed[0]["response"]
|
||||
assert isinstance(final, dict) and _output_text(final) == _answer(marker), final
|
||||
deltas: Final = "".join(str(event["delta"]) for event in events if event["type"] == "response.output_text.delta")
|
||||
assert deltas == _answer(marker), events
|
||||
|
||||
|
||||
def _posted(wire: Wire) -> tuple[Request, ...]:
|
||||
return tuple(request for request in wire.drain() if request.method == "POST")
|
||||
|
||||
|
||||
def _only_post(wire: Wire, target: str) -> dict[str, JsonValue]:
|
||||
posted: Final = _posted(wire)
|
||||
assert [request.target for request in posted] == [target], posted
|
||||
return _JSON_OBJECT.validate_json(posted[0].body)
|
||||
|
||||
|
||||
def _chat_messages(marker: str) -> list[JsonValue]:
|
||||
return [{"role": "user", "content": _prompt(marker)}]
|
||||
|
||||
|
||||
def _unsupported_param_error(response: httpx.Response) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
error: Final = _JSON_OBJECT.validate_json(response.text)["error"]
|
||||
assert isinstance(error, dict) and "verbosity" in str(error["message"]), response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
def test_regression_explicit_bridge_sends_text_verbosity_as_top_level_verbosity(
|
||||
gateway: Gateway, verbosity: Verbosity
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker, text={"verbosity": verbosity}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {"model": _GPT, "messages": _chat_messages(marker), "verbosity": verbosity}, sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
def test_regression_explicit_bridge_stream_sends_text_verbosity_as_top_level_verbosity(
|
||||
gateway: Gateway, verbosity: Verbosity
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": verbosity}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {
|
||||
"model": _GPT,
|
||||
"messages": _chat_messages(marker),
|
||||
"stream": True,
|
||||
**_STREAM_OPTIONS,
|
||||
"verbosity": verbosity,
|
||||
}, sent
|
||||
|
||||
|
||||
def test_regression_explicit_bridge_sends_verbosity_beside_the_json_schema_response_format(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {
|
||||
"model": _GPT,
|
||||
"messages": _chat_messages(marker),
|
||||
"response_format": _RESPONSE_FORMAT,
|
||||
"verbosity": "low",
|
||||
}, sent
|
||||
|
||||
|
||||
def test_regression_automatic_bridge_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None:
|
||||
refused: Final = uuid.uuid4().hex
|
||||
control: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=False)
|
||||
_unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "low"}))
|
||||
_answered(_responses(gateway, model, control), control)
|
||||
posted: Final = _posted(wire)
|
||||
assert [(request.target, _marker_of(request)) for request in posted] == [
|
||||
("/serving-endpoints/chat/completions", control)
|
||||
], posted
|
||||
|
||||
|
||||
def test_regression_automatic_bridge_stream_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None:
|
||||
refused: Final = uuid.uuid4().hex
|
||||
control: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=False)
|
||||
_unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "high"}, stream=True))
|
||||
_answered(_responses(gateway, model, control), control)
|
||||
posted: Final = _posted(wire)
|
||||
assert [(request.target, _marker_of(request)) for request in posted] == [
|
||||
("/serving-endpoints/chat/completions", control)
|
||||
], posted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["deployment", "request"])
|
||||
def test_regression_allowed_openai_params_forwards_verbosity_through_the_automatic_bridge(
|
||||
gateway: Gateway, source: Literal["deployment", "request"]
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
allowed: Final[dict[str, JsonValue]] = {"allowed_openai_params": ["verbosity"]}
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=False, **(allowed if source == "deployment" else {}))
|
||||
extra: Final = allowed if source == "request" else {}
|
||||
_answered(_responses(gateway, model, marker, text={"verbosity": "low"}, **extra), marker)
|
||||
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
|
||||
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "verbosity": "low"}, sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["deployment", "request"])
|
||||
def test_guard_drop_params_drops_unsupported_verbosity_and_keeps_the_json_schema_format(
|
||||
gateway: Gateway, source: Literal["deployment", "request"]
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=source == "deployment")
|
||||
extra: Final[dict[str, JsonValue]] = {"drop_params": True} if source == "request" else {}
|
||||
_answered(
|
||||
_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}, **extra), marker
|
||||
)
|
||||
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
|
||||
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent
|
||||
|
||||
|
||||
def test_guard_drop_params_stream_drops_unsupported_verbosity(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=True)
|
||||
_streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": "high"}), marker)
|
||||
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
|
||||
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "stream": True}, sent
|
||||
|
||||
|
||||
def test_guard_explicit_bridge_without_verbosity_sends_only_the_json_schema_format(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {"model": _GPT, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent
|
||||
|
||||
|
||||
def test_guard_explicit_bridge_without_text_sends_no_verbosity(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {"model": _GPT, "messages": _chat_messages(marker)}, sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
|
||||
def test_guard_native_responses_route_forwards_text_verbosity_unchanged(gateway: Gateway, stream: bool) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
text: Final[dict[str, JsonValue]] = {"format": _TEXT_FORMAT, "verbosity": "high"}
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _native_gpt(scenario, wire)
|
||||
if stream:
|
||||
_streamed_answer(_streamed_events(gateway, model, marker, text=text), marker)
|
||||
else:
|
||||
_answered(_responses(gateway, model, marker, text=text), marker)
|
||||
sent: Final = _only_post(wire, "/v1/responses")
|
||||
assert sent == {"model": _GPT, "input": _prompt(marker), "text": text, **({"stream": True} if stream else {})}, sent
|
||||
|
||||
|
||||
_CLIENT_TOOL_SEARCH: Final[dict[str, JsonValue]] = {"type": "tool_search", "execution": "client"}
|
||||
_CLIENT_TOOL_SEARCH_ON_THE_WIRE: Final[dict[str, JsonValue]] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "tool_search",
|
||||
"description": (
|
||||
"Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query for deferred tools."},
|
||||
"limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."},
|
||||
},
|
||||
"required": ["query"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FollowUpWire:
|
||||
sent: JsonValue
|
||||
expected: dict[str, JsonValue]
|
||||
|
||||
|
||||
def _stored_prompts_config(directory: Path) -> Path:
|
||||
stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
general: Final = _JSON_OBJECT.validate_python(stock["general_settings"])
|
||||
path: Final = directory / "verbosity-stored-turns.yaml"
|
||||
path.write_text(yaml.safe_dump({**stock, "general_settings": {**general, "store_prompts_in_spend_logs": True}}))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def stored_turns_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[OwnedProxy, Wire]]:
|
||||
directory: Final = tmp_path_factory.mktemp("verbosity-stored-turns")
|
||||
with _upstream() as wire, gateway_from_environment() as gateway:
|
||||
with owned_proxy_process(gateway, directory, {}, config=_stored_prompts_config(directory)) as owned:
|
||||
yield owned, wire
|
||||
|
||||
|
||||
def _stored_first_turn(proxy: Gateway, model: str, marker: str) -> str:
|
||||
first: Final = _responses(proxy, model, marker, tools=[_CLIENT_TOOL_SEARCH])
|
||||
_answered(first, marker)
|
||||
response_id: Final = str(_JSON_OBJECT.validate_json(first.text)["id"])
|
||||
eventually(
|
||||
lambda: proxy.rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
|
||||
lambda rows: any(rv.same_response(str(row["request_id"]), response_id) for row in rows),
|
||||
seconds=70,
|
||||
)
|
||||
return response_id
|
||||
|
||||
|
||||
def _client_tool_search_follow_up(
|
||||
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, **fields: JsonValue
|
||||
) -> _FollowUpWire:
|
||||
owned, wire = stored_turns_proxy
|
||||
proxy: Final = owned.gateway
|
||||
first: Final = uuid.uuid4().hex
|
||||
second: Final = uuid.uuid4().hex
|
||||
with proxy.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
previous: Final = _stored_first_turn(proxy, model, first)
|
||||
wire.drain()
|
||||
follow_up: Final[dict[str, JsonValue]] = {
|
||||
"previous_response_id": previous,
|
||||
"tools": [_CLIENT_TOOL_SEARCH],
|
||||
**fields,
|
||||
}
|
||||
if stream:
|
||||
_streamed_answer(_streamed_events(proxy, model, second, **follow_up), second)
|
||||
else:
|
||||
_answered(_responses(proxy, model, second, **follow_up), second)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
return _FollowUpWire(
|
||||
sent,
|
||||
{
|
||||
"model": _GPT,
|
||||
"messages": [
|
||||
*_chat_messages(first),
|
||||
{"role": "assistant", "content": _answer(first)},
|
||||
*_chat_messages(second),
|
||||
],
|
||||
"tools": [_CLIENT_TOOL_SEARCH_ON_THE_WIRE],
|
||||
**({"stream": True, **_STREAM_OPTIONS} if stream else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
def test_regression_client_tool_search_follow_up_sends_text_verbosity_as_top_level_verbosity(
|
||||
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, verbosity: Verbosity
|
||||
) -> None:
|
||||
observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream, text={"verbosity": verbosity})
|
||||
assert observed.sent == {**observed.expected, "verbosity": verbosity}, observed.sent
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
|
||||
def test_guard_client_tool_search_follow_up_without_verbosity_sends_no_verbosity(
|
||||
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool
|
||||
) -> None:
|
||||
observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream)
|
||||
assert observed.sent == observed.expected, observed.sent
|
||||
|
|
@ -1595,3 +1595,57 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search
|
|||
[("tool_result", "toolu_standup"), ("text", None)],
|
||||
]
|
||||
assert prisma_client.db.queries == 1
|
||||
|
||||
|
||||
async def _codex_follow_up_body(**kwargs: object) -> Mapping[str, object]:
|
||||
anthropic: Final = _RecordingRawAnthropicMessages()
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])):
|
||||
await litellm.aresponses(
|
||||
model=_MODEL,
|
||||
previous_response_id="chatcmpl-calendar-turn",
|
||||
input=[{"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}],
|
||||
tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL],
|
||||
api_key="sk-ant-fake",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)),
|
||||
**kwargs,
|
||||
)
|
||||
return json.loads(anthropic.body)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_tool_search_follow_up_rejects_text_verbosity_anthropic_does_not_support(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
anthropic: Final = _RecordingRawAnthropicMessages()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])),
|
||||
pytest.raises(litellm.UnsupportedParamsError, match="verbosity"),
|
||||
):
|
||||
await litellm.aresponses(
|
||||
model=_MODEL,
|
||||
previous_response_id="chatcmpl-calendar-turn",
|
||||
input=[{"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}],
|
||||
tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL],
|
||||
text={"verbosity": "low"},
|
||||
api_key="sk-ant-fake",
|
||||
num_retries=0,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)),
|
||||
)
|
||||
|
||||
assert anthropic.body == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_tool_search_follow_up_text_verbosity_follows_provider_param_validation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
without_text: Final = await _codex_follow_up_body()
|
||||
|
||||
dropped: Final = await _codex_follow_up_body(text={"verbosity": "low"}, drop_params=True)
|
||||
allowed: Final = await _codex_follow_up_body(text={"verbosity": "low"}, allowed_openai_params=["verbosity"])
|
||||
|
||||
assert dropped == without_text
|
||||
assert allowed == {**without_text, "verbosity": "low"}
|
||||
|
|
|
|||
|
|
@ -554,7 +554,6 @@ class TestUseResponsesApiBridgeFlag:
|
|||
truncation="auto",
|
||||
metadata={"k": "v"},
|
||||
background=False,
|
||||
text={"verbosity": "low"},
|
||||
api_base="https://example.databricks.test/serving-endpoints",
|
||||
api_key="fake-databricks-api-key",
|
||||
num_retries=0,
|
||||
|
|
@ -1103,3 +1102,281 @@ async def test_chatgpt_completion_routes_to_responses_without_a_provider_catalog
|
|||
def test_missing_catalog_entry_does_not_force_other_providers_to_responses(provider: str) -> None:
|
||||
info, model = responses_api_bridge_check(model="test-chatgpt-routing-control", custom_llm_provider=provider)
|
||||
assert (info.get("mode"), model) == (None, "test-chatgpt-routing-control")
|
||||
|
||||
|
||||
_GPT_5_6_CHUNKS: Final = (
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "Answer"}, "finish_reason": None}],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _openai_chat_upstream(respx_mock: respx.MockRouter, stream: bool) -> respx.Route:
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions")
|
||||
if stream:
|
||||
return route.respond(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
text="".join(f"data: {json.dumps(chunk)}\n\n" for chunk in _GPT_5_6_CHUNKS) + "data: [DONE]\n\n",
|
||||
)
|
||||
return route.respond(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _bridged_responses_call(asynchronous: bool, stream: bool, **kwargs: object) -> None:
|
||||
if not asynchronous:
|
||||
response = litellm.responses(
|
||||
input="Hello", stream=stream, api_key="fake-provider-api-key", num_retries=0, **kwargs
|
||||
)
|
||||
_ = list(response) if stream else response
|
||||
return
|
||||
response = await litellm.aresponses(
|
||||
input="Hello", stream=stream, api_key="fake-provider-api-key", num_retries=0, **kwargs
|
||||
)
|
||||
_ = [event async for event in response] if stream else response
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_sends_text_verbosity_as_chat_verbosity(
|
||||
verbosity: str, asynchronous: bool, stream: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
upstream: Final = _openai_chat_upstream(respx_mock, stream)
|
||||
|
||||
await _bridged_responses_call(
|
||||
asynchronous, stream, model="openai/gpt-5.6", use_chat_completions_api=True, text={"verbosity": verbosity}
|
||||
)
|
||||
|
||||
assert upstream.call_count == 1
|
||||
assert json.loads(upstream.calls[0].request.read()) == {
|
||||
"model": "gpt-5.6",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"verbosity": verbosity,
|
||||
**({"stream": True, "stream_options": {"include_usage": True}} if stream else {}),
|
||||
}
|
||||
|
||||
|
||||
_JSON_SCHEMA_TEXT_FORMAT: Final = {
|
||||
"type": "json_schema",
|
||||
"name": "answer",
|
||||
"schema": {"type": "object", "properties": {"answer": {"type": "string"}}},
|
||||
"strict": True,
|
||||
}
|
||||
_JSON_SCHEMA_RESPONSE_FORMAT: Final = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {"type": "object", "properties": {"answer": {"type": "string"}}},
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected_fields"),
|
||||
[
|
||||
pytest.param(
|
||||
{"verbosity": "low", "format": _JSON_SCHEMA_TEXT_FORMAT},
|
||||
{"response_format": _JSON_SCHEMA_RESPONSE_FORMAT, "verbosity": "low"},
|
||||
id="verbosity-and-format",
|
||||
),
|
||||
pytest.param(
|
||||
{"format": _JSON_SCHEMA_TEXT_FORMAT}, {"response_format": _JSON_SCHEMA_RESPONSE_FORMAT}, id="format-only"
|
||||
),
|
||||
pytest.param(None, {}, id="no-text"),
|
||||
],
|
||||
)
|
||||
def test_bridge_maps_text_format_alongside_verbosity(
|
||||
text: dict[str, object] | None,
|
||||
expected_fields: dict[str, object],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
upstream: Final = _openai_chat_upstream(respx_mock, stream=False)
|
||||
|
||||
litellm.responses(
|
||||
model="openai/gpt-5.6",
|
||||
input="Hello",
|
||||
use_chat_completions_api=True,
|
||||
api_key="fake-provider-api-key",
|
||||
num_retries=0,
|
||||
**({"text": text} if text is not None else {}),
|
||||
)
|
||||
|
||||
assert upstream.call_count == 1
|
||||
assert json.loads(upstream.calls[0].request.read()) == {
|
||||
"model": "gpt-5.6",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
**expected_fields,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_rejects_text_verbosity_the_provider_does_not_support_when_drop_params_is_off(
|
||||
asynchronous: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
upstream: Final = _databricks_chat_upstream(respx_mock)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as raised:
|
||||
await _bridged_responses_call(
|
||||
asynchronous,
|
||||
False,
|
||||
model="databricks/my-custom-model",
|
||||
text={"verbosity": "low"},
|
||||
api_base="https://example.databricks.test/serving-endpoints",
|
||||
)
|
||||
|
||||
assert upstream.call_count == 0, "the provider must not be called with a verbosity it does not support"
|
||||
assert raised.value.status_code == 400
|
||||
assert "verbosity" in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call_kwargs", "global_drop_params", "expected_fields"),
|
||||
[
|
||||
pytest.param({"drop_params": True}, False, {}, id="request-drop-params"),
|
||||
pytest.param({}, True, {}, id="global-drop-params"),
|
||||
pytest.param({"allowed_openai_params": ["verbosity"]}, False, {"verbosity": "low"}, id="allowed-openai-params"),
|
||||
],
|
||||
)
|
||||
def test_bridge_text_verbosity_follows_provider_param_validation(
|
||||
call_kwargs: dict[str, object],
|
||||
global_drop_params: bool,
|
||||
expected_fields: dict[str, object],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", global_drop_params)
|
||||
upstream: Final = _databricks_chat_upstream(respx_mock)
|
||||
|
||||
response: Final = litellm.responses(
|
||||
model="databricks/my-custom-model",
|
||||
input="Hello",
|
||||
text={"verbosity": "low"},
|
||||
api_base="https://example.databricks.test/serving-endpoints",
|
||||
api_key="fake-databricks-api-key",
|
||||
num_retries=0,
|
||||
**call_kwargs,
|
||||
)
|
||||
|
||||
assert upstream.call_count == 1
|
||||
assert json.loads(upstream.calls[0].request.read()) == {
|
||||
"model": "my-custom-model",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
**expected_fields,
|
||||
}
|
||||
assert response.output[0].content[0].text == "Answer"
|
||||
|
||||
|
||||
def test_native_responses_route_keeps_text_verbosity_nested(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
litellm.responses(
|
||||
model="openai/gpt-5.6",
|
||||
input="Hello",
|
||||
text={"verbosity": "low"},
|
||||
api_key="fake-provider-api-key",
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
assert upstream.call_count == 1
|
||||
assert json.loads(upstream.calls[0].request.read()) == {
|
||||
"model": "gpt-5.6",
|
||||
"input": "Hello",
|
||||
"text": {"verbosity": "low"},
|
||||
}
|
||||
|
||||
|
||||
_CLIENT_TOOL_SEARCH: Final = {
|
||||
"type": "tool_search",
|
||||
"execution": "client",
|
||||
"description": "Search the deferred app tools.",
|
||||
"parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_sends_text_verbosity_on_a_client_tool_search_follow_up_turn(
|
||||
verbosity: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
upstream: Final = _openai_chat_upstream(respx_mock, stream=False)
|
||||
|
||||
await _bridged_responses_call(
|
||||
True,
|
||||
False,
|
||||
model="openai/gpt-5.6",
|
||||
use_chat_completions_api=True,
|
||||
previous_response_id="chatcmpl-earlier-turn",
|
||||
tools=[_CLIENT_TOOL_SEARCH],
|
||||
text={"verbosity": verbosity},
|
||||
)
|
||||
|
||||
assert upstream.call_count == 1
|
||||
assert json.loads(upstream.calls[0].request.read()) == {
|
||||
"model": "gpt-5.6",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "tool_search",
|
||||
"description": _CLIENT_TOOL_SEARCH["description"],
|
||||
"parameters": _CLIENT_TOOL_SEARCH["parameters"],
|
||||
},
|
||||
}
|
||||
],
|
||||
"verbosity": verbosity,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue