mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test(integration): cover comparison_response_id unwrap on the responses wire tests
This commit is contained in:
parent
cd36284138
commit
7faad8fdc9
2 changed files with 104 additions and 2 deletions
|
|
@ -1,12 +1,21 @@
|
|||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually, gateway_from_environment, object_value, string_value
|
||||
import websockets
|
||||
from integration._support.client import (
|
||||
Gateway,
|
||||
Scenario,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.responses_stream import RATE_LIMIT_MESSAGE, openai_error_event
|
||||
|
|
@ -14,17 +23,21 @@ from integration._support.responses_ws_peer import (
|
|||
PROVIDER_MODEL,
|
||||
USAGE,
|
||||
completed,
|
||||
create,
|
||||
drain,
|
||||
events,
|
||||
responses_peer,
|
||||
session,
|
||||
turn,
|
||||
)
|
||||
from integration._support.tls import write_self_signed_cert
|
||||
from pydantic import JsonValue
|
||||
from integration._support.worker_readiness import wait_for_worker_readiness
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 120), pytest.mark.shares_worker_state]
|
||||
PROXY_OPENAI_KEY: Final = "sk-proxy-environment-openai-key"
|
||||
PROXY_AZURE_KEY: Final = "proxy-environment-azure-key"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _top_level_rate_limit(response_id: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
|
|
@ -89,6 +102,68 @@ def test_previous_response_id_from_the_first_turn_reaches_the_provider_as_its_ow
|
|||
assert forwarded[1]["previous_response_id"] == "resp_peer_1"
|
||||
|
||||
|
||||
def _enable_responses_affinity(gateway: Gateway, scenario: Scenario) -> None:
|
||||
settings: Final = object_value(gateway.get("/router/settings")["current_values"])
|
||||
current: Final = settings.get("optional_pre_call_checks")
|
||||
original: Final = list(current) if isinstance(current, list) else []
|
||||
scenario.cleanups.callback(
|
||||
gateway.post, "/config/update", {"router_settings": {"optional_pre_call_checks": original}}
|
||||
)
|
||||
gateway.post(
|
||||
"/config/update",
|
||||
{"router_settings": {"optional_pre_call_checks": [*original, "responses_api_deployment_check"]}},
|
||||
)
|
||||
|
||||
|
||||
def _add_preferred_deployment(gateway: Gateway, scenario: Scenario, group: str, api_base: str) -> None:
|
||||
created: Final = gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": f"openai/{PROVIDER_MODEL}", "api_base": api_base, "order": 1},
|
||||
"model_info": {},
|
||||
},
|
||||
)
|
||||
scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))
|
||||
wait_for_worker_readiness(gateway, group, 2, gateway.workers)
|
||||
|
||||
|
||||
async def _compare_on_a_new_socket(
|
||||
proxy_url: str, key: str, model: str, compared_id: str
|
||||
) -> tuple[dict[str, JsonValue], ...]:
|
||||
proxy: Final = proxy_url.rstrip("/").replace("http://", "ws://")
|
||||
frame: Final = {
|
||||
**_JSON_OBJECT.validate_json(create(model, f"compare seven {uuid.uuid4().hex}")),
|
||||
"prompt_cache_options": {"comparison_response_id": compared_id},
|
||||
}
|
||||
async with websockets.connect(
|
||||
f"{proxy}/v1/responses?model={model}", additional_headers={"Authorization": f"Bearer {key}"}, open_timeout=10
|
||||
) as connection:
|
||||
return await turn(connection, json.dumps(frame))
|
||||
|
||||
|
||||
def test_a_first_frame_comparing_an_earlier_response_reaches_its_deployment_with_the_providers_own_id(
|
||||
candidate: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as compared_peer, responses_peer(cert) as other_peer, candidate.scenario() as scenario:
|
||||
_enable_responses_affinity(candidate, scenario)
|
||||
model: Final = scenario.model(model=f"openai/{PROVIDER_MODEL}", api_base=compared_peer.url)
|
||||
key: Final = scenario.key(models=[model])
|
||||
base_url: Final = str(candidate.client.base_url)
|
||||
(first,) = asyncio.run(session(base_url, key, model, (f"remember seven {uuid.uuid4().hex}",)))
|
||||
compared_id: Final = string_value(completed(first)["id"])
|
||||
assert compared_id.startswith("resp_") and compared_id != "resp_peer_1", compared_id
|
||||
_add_preferred_deployment(candidate, scenario, model, other_peer.url)
|
||||
second: Final = asyncio.run(_compare_on_a_new_socket(base_url, key, model, compared_id))
|
||||
assert completed(second)["status"] == "completed"
|
||||
assert drain(other_peer.frames) == (), "comparison_response_id did not pin the compared deployment"
|
||||
forwarded: Final = drain(compared_peer.frames)
|
||||
assert [frame.get("prompt_cache_options") for frame in forwarded] == [
|
||||
None,
|
||||
{"comparison_response_id": "resp_peer_1"},
|
||||
], forwarded
|
||||
|
||||
|
||||
def test_a_top_level_error_frame_reaches_the_client_and_logs_the_provider_code_and_message(
|
||||
candidate: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -229,6 +229,33 @@ def test_input_tokens_count_goes_to_the_deployment_that_served_the_previous_resp
|
|||
], [request.body for request in requests]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nested", (False, True), ids=("top-level", "extra-body"))
|
||||
def test_comparison_response_id_uses_the_raw_id_and_affinity(gateway: Gateway, nested: bool) -> None:
|
||||
compared_raw_id: Final = f"resp_compared_{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.method == "GET" and request.target == "/v1/models":
|
||||
return model_discovery_reply(_MODEL)
|
||||
return Reply(body=_response(compared_raw_id))
|
||||
|
||||
with wire_server(respond) as first_wire, wire_server(respond) as second_wire, gateway.scenario() as scenario:
|
||||
model: Final = f"integration-{uuid.uuid4().hex}"
|
||||
_enable_responses_affinity(scenario)
|
||||
assert _register(scenario, first_wire, model) == model
|
||||
client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
|
||||
compared: Final = client.responses.create(model=model, input="turn one")
|
||||
assert compared.id.startswith("resp_") and compared.id != compared_raw_id, compared.model_dump_json()
|
||||
_register_in_group(scenario, second_wire, model, order=1)
|
||||
options: Final = {"prompt_cache_options": {"comparison_response_id": compared.id}}
|
||||
client.responses.create(model=model, input="turn two", extra_body={"extra_body": options} if nested else options)
|
||||
assert drain_contract_requests(second_wire) == (), "comparison_response_id did not pin the compared deployment"
|
||||
requests: Final = drain_contract_requests(first_wire)
|
||||
assert [_JSON_OBJECT.validate_json(request.body) for request in requests] == [
|
||||
{"model": _MODEL, "input": "turn one"},
|
||||
{"model": _MODEL, "input": "turn two", "prompt_cache_options": {"comparison_response_id": compared_raw_id}},
|
||||
], [request.body for request in requests]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", _ALIASES)
|
||||
def test_codex_responses_request_preserves_all_supported_fields(gateway: Gateway, prefix: str) -> None:
|
||||
response_id: Final = f"resp_codex_{uuid.uuid4().hex}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue