mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
test: cover response contracts across translation endpoints (#46632)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu
GitHub
parent
22a8cd0cb7
commit
4496fe0ebc
10 changed files with 1021 additions and 4 deletions
|
|
@ -13,6 +13,8 @@ from integration._support.client import Gateway, eventually
|
|||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.translation.chat_completions.cross_model.contracts import assert_completion
|
||||
from integration.translation.response_contracts import responses_structure
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "databricks-glm-5-2"
|
||||
|
|
@ -81,6 +83,43 @@ _JSON_TOOL_CALL_MESSAGE: Final[dict[str, JsonValue]] = {
|
|||
}
|
||||
|
||||
|
||||
def test_databricks_responses_reasoning_summary_keeps_reasoning_effort_a_string(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/chat/completions"
|
||||
assert _JSON_OBJECT.validate_json(request.body) == {
|
||||
"model": _BACKEND,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"max_tokens": 64,
|
||||
"reasoning_effort": "low",
|
||||
}
|
||||
return Reply(body=_chat_completion_response(_BACKEND, {"role": "assistant", "content": "Hello."}, "stop"))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
alias: Final = scenario.model(model=f"databricks/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": alias,
|
||||
"input": "Say hello.",
|
||||
"max_output_tokens": 64,
|
||||
"reasoning": {"effort": "low", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
responses_structure(body)
|
||||
assert body["usage"] == {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
"input_tokens_details": None,
|
||||
"output_tokens_details": None,
|
||||
"cost": None,
|
||||
}
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def _chat_completion_response(model: str, message: dict[str, JsonValue], finish_reason: str) -> bytes:
|
||||
response: Final = {
|
||||
"id": "databricks-json-schema-response",
|
||||
|
|
@ -455,6 +494,7 @@ def test_databricks_json_schema_refs_are_preserved_in_chat_completions(gateway:
|
|||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert_completion(body)
|
||||
assert body["choices"] == [
|
||||
{
|
||||
"index": 0,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import pytest
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from integration.translation.chat_completions.cross_model.contracts import assert_completion
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "qwen3-reasoning"
|
||||
|
|
@ -186,6 +187,7 @@ def test_hosted_vllm_assistant_reasoning_content_reaches_the_wire(gateway: Gatew
|
|||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert_completion(payload)
|
||||
assert payload["id"] == identity
|
||||
_only_request(wire)
|
||||
|
||||
|
|
|
|||
|
|
@ -336,6 +336,18 @@ model_list:
|
|||
model: openai/responses/gpt-6.1-sol
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-openai-key
|
||||
- model_name: contract-affinity-responses
|
||||
litellm_params:
|
||||
model: openai/responses/gpt-6.1-sol
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-openai-key
|
||||
provider_affinity_header: x-session-affinity
|
||||
- model_name: contract-affinity-anthropic
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-5-5
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-anthropic-key
|
||||
provider_affinity_header: x-session-affinity
|
||||
- model_name: azure/gpt-5.4
|
||||
litellm_params:
|
||||
model: azure/gpt-5.4
|
||||
|
|
|
|||
|
|
@ -0,0 +1,237 @@
|
|||
import json
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT, Gateway, object_value
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration._support.wire import Reply
|
||||
from integration.translation.case import SSEFrame, StreamingTranslationTestCase
|
||||
from integration.translation.chat_completions.bases.openai_responses import GPT_6_1_SOL_TEST_CASE
|
||||
from integration.translation.chat_completions.cross_model.contracts import (
|
||||
Message,
|
||||
assert_completion,
|
||||
assert_final_text,
|
||||
assert_stream_preserved,
|
||||
)
|
||||
from integration.translation.response_scenarios import text_completion, two_messages_before_tools
|
||||
from integration.translation.runner import (
|
||||
assert_provider_request,
|
||||
assert_translation,
|
||||
parse_sse_frames,
|
||||
streaming_reply,
|
||||
)
|
||||
from openai.lib.streaming.chat import ChatCompletionStreamState
|
||||
from openai.types.chat import ChatCompletionChunk
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
OBJECTS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
|
||||
GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE)
|
||||
GPT_6_1_SOL_STRING_TOOL_OUTPUT_TEST_CASE: Final = replace(
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
scenario="plain_string_tool_output",
|
||||
litellm_request={
|
||||
**GPT_6_1_SOL_TEST_CASE.litellm_request,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Read the validation file."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_contract_1",
|
||||
"type": "function",
|
||||
"function": {"name": "read_file", "arguments": '{"path":"shop/validation.py"}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_contract_1", "content": '{"result":"quantity must be positive"}'},
|
||||
],
|
||||
},
|
||||
expected_provider_request={
|
||||
**{
|
||||
key: value
|
||||
for key, value in GPT_6_1_SOL_TEST_CASE.expected_provider_request.items()
|
||||
if key != "instructions"
|
||||
},
|
||||
"input": [
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "Read the validation file."}]},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_contract_1",
|
||||
"name": "read_file",
|
||||
"arguments": '{"path":"shop/validation.py"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_contract_1",
|
||||
"output": '{"result":"quantity must be positive"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_multiple_responses_messages_and_tools_produce_one_complete_choice(
|
||||
gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
body: Final = assert_translation(GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE, gateway, provider)
|
||||
assert_completion(body)
|
||||
|
||||
|
||||
def test_chat_to_responses_keeps_plain_string_tool_outputs_as_strings(
|
||||
gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
pytest.skip("BUG: LIT-9038 plain-string tool outputs are wrapped in a list that strict Responses backends reject")
|
||||
body: Final = assert_translation(GPT_6_1_SOL_STRING_TOOL_OUTPUT_TEST_CASE, gateway, provider)
|
||||
assert_completion(body)
|
||||
|
||||
|
||||
def test_azure_deployment_capability_override_filters_temperature_even_with_base_model(
|
||||
gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
alias: Final = scenario.model(
|
||||
model="azure/contract-deployment",
|
||||
api_base=provider.wire.url,
|
||||
api_key="synthetic-azure-key",
|
||||
api_version="2025-04-01-preview",
|
||||
base_model="azure/gpt-6.1-sol",
|
||||
drop_params=True,
|
||||
model_info={"supports_none_reasoning_effort": False},
|
||||
)
|
||||
provider.expect(Reply(body=json.dumps(text_completion("gpt-6.1-sol")).encode()))
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": alias,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"max_tokens": 64,
|
||||
"temperature": 0.2,
|
||||
"use_chat_completions_api": True,
|
||||
},
|
||||
)
|
||||
assert_provider_request(
|
||||
provider.received(),
|
||||
"/openai/deployments/contract-deployment/chat/completions?api-version=2025-04-01-preview",
|
||||
{
|
||||
"content-type": "application/json",
|
||||
"api-key": "synthetic-azure-key",
|
||||
"authorization": "Bearer synthetic-azure-key",
|
||||
},
|
||||
{
|
||||
"model": "contract-deployment",
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"max_completion_tokens": 64,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {
|
||||
**text_completion(alias),
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello."},
|
||||
"finish_reason": "stop",
|
||||
"provider_specific_fields": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
assert_final_text(JSON_OBJECT.validate_json(response.content))
|
||||
|
||||
|
||||
def _cached_stream(missing_index: bool, terminal_choices: bool) -> StreamingTranslationTestCase:
|
||||
index: Final[dict[str, JsonValue]] = {} if missing_index else {"index": 0}
|
||||
envelope: Final[dict[str, JsonValue]] = {
|
||||
"id": "chatcmpl-cached-stream",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-6.1-sol",
|
||||
}
|
||||
usage: Final[dict[str, JsonValue]] = {
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 100,
|
||||
"total_tokens": 1100,
|
||||
"prompt_tokens_details": {"cached_tokens": 600},
|
||||
}
|
||||
finish: Final[dict[str, JsonValue]] = {**index, "delta": {}, "finish_reason": "stop"}
|
||||
return StreamingTranslationTestCase(
|
||||
scenario="choice_bearing_cached_usage" if terminal_choices else "usage_only_cached_usage",
|
||||
litellm_endpoint="/v1/chat/completions",
|
||||
litellm_request={
|
||||
"model": "openai/gpt-6.1-sol",
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"max_tokens": 64,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
"use_chat_completions_api": True,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/chat/completions",
|
||||
expected_provider_headers=GPT_6_1_SOL_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_request={
|
||||
"model": "gpt-6.1-sol",
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"max_completion_tokens": 64,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
mock_provider_frames=(
|
||||
SSEFrame(
|
||||
data={
|
||||
**envelope,
|
||||
"choices": [{**index, "delta": {"role": "assistant", "content": "Hello."}, "finish_reason": None}],
|
||||
}
|
||||
),
|
||||
*(() if terminal_choices else (SSEFrame(data={**envelope, "choices": [finish]}),)),
|
||||
SSEFrame(data={**envelope, "choices": [finish] if terminal_choices else [], "usage": usage}),
|
||||
SSEFrame(data="[DONE]"),
|
||||
),
|
||||
expected_litellm_message={"role": "assistant", "content": "Hello."},
|
||||
expected_litellm_usage={
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 100,
|
||||
"total_tokens": 1100,
|
||||
"prompt_tokens_details": {"cached_tokens": 600},
|
||||
"completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 100},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_index", (False, True), ids=("indexed", "omitted_index"))
|
||||
@pytest.mark.parametrize("terminal_choices", (False, True), ids=("usage_only", "choices_and_usage"))
|
||||
def test_cached_usage_survives_raw_streaming_and_sdk_assembly(
|
||||
missing_index: bool, terminal_choices: bool, gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
if missing_index:
|
||||
pytest.skip("BUG: LIT-9468 omitted upstream choice indexes reach the SDK as null and abort the stream")
|
||||
case: Final = _cached_stream(missing_index, terminal_choices)
|
||||
provider.expect(streaming_reply(case))
|
||||
response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request)
|
||||
assert_provider_request(
|
||||
provider.received(),
|
||||
case.expected_provider_endpoint,
|
||||
case.expected_provider_headers,
|
||||
case.expected_provider_request,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
frames: Final = parse_sse_frames(response.text)
|
||||
assert frames and tuple(frame.data == "[DONE]" for frame in frames) == (False,) * (len(frames) - 1) + (True,)
|
||||
chunks: Final = tuple(JSON_OBJECT.validate_python(frame.data) for frame in frames[:-1])
|
||||
state: Final = ChatCompletionStreamState()
|
||||
for chunk in chunks:
|
||||
assert chunk["object"] == "chat.completion.chunk", chunk
|
||||
for choice in OBJECTS.validate_python(chunk["choices"]):
|
||||
assert type(choice["index"]) is int and choice["index"] == 0, choice
|
||||
state.handle_chunk(ChatCompletionChunk.model_validate(chunk))
|
||||
body: Final = JSON_OBJECT.validate_json(state.get_final_completion().model_dump_json())
|
||||
assert_final_text(body)
|
||||
message: Final = Message.model_validate(object_value(OBJECTS.validate_python(body["choices"])[0]["message"]))
|
||||
usage: Final = object_value(body["usage"])
|
||||
assert_stream_preserved(message, usage, case)
|
||||
raw_usage: Final = tuple(chunk["usage"] for chunk in chunks if chunk.get("usage"))
|
||||
assert len(raw_usage) == 1, response.text
|
||||
assert_stream_preserved(message, object_value(raw_usage[0]), case)
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
import json
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT, Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration._support.wire import Reply
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE
|
||||
from integration.translation.messages.bases.azure_ai import (
|
||||
CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
)
|
||||
from integration.translation.messages.bases.openai_responses import GPT_6_1_SOL_TEST_CASE
|
||||
from integration.translation.response_contracts import messages_structure
|
||||
from integration.translation.response_scenarios import text_completion, two_messages_before_tools
|
||||
from integration.translation.runner import assert_provider_request, assert_translation
|
||||
|
||||
GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE)
|
||||
AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace(
|
||||
AZURE_CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
scenario="cross_model_rich_request",
|
||||
litellm_request={**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, "model": "azure_ai/claude-sonnet-5-5"},
|
||||
expected_provider_request={
|
||||
**CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request,
|
||||
"system": [{"type": "text", "text": CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request["system"]}],
|
||||
},
|
||||
mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response,
|
||||
expected_litellm_response={
|
||||
**CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response,
|
||||
"model": "azure_ai/claude-sonnet-5-5",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
(AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE, GPT_6_1_SOL_TEST_CASE),
|
||||
ids=(AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE.id, GPT_6_1_SOL_TEST_CASE.id),
|
||||
)
|
||||
def test_messages_keep_the_same_public_contract_on_native_and_bridged_routes(
|
||||
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert {key: value for key, value in case.litellm_request.items() if key != "model"} == {
|
||||
key: value for key, value in CLAUDE_SONNET_5_5_TEST_CASE.litellm_request.items() if key != "model"
|
||||
}
|
||||
native: Final = assert_translation(CLAUDE_SONNET_5_5_TEST_CASE, gateway, provider)
|
||||
compared: Final = assert_translation(case, gateway, provider)
|
||||
assert messages_structure(compared) == messages_structure(native)
|
||||
|
||||
|
||||
def test_messages_preserves_multiple_responses_messages_before_tools(
|
||||
gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
body: Final = assert_translation(GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE, gateway, provider)
|
||||
messages_structure(body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
(CLAUDE_SONNET_5_5_TEST_CASE, GPT_6_1_SOL_TEST_CASE),
|
||||
ids=(CLAUDE_SONNET_5_5_TEST_CASE.id, GPT_6_1_SOL_TEST_CASE.id),
|
||||
)
|
||||
def test_messages_forwards_the_configured_provider_affinity_header_on_native_and_bridged_routes(
|
||||
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
if case is CLAUDE_SONNET_5_5_TEST_CASE:
|
||||
pytest.skip("BUG: LIT-8405 native Messages drops the configured provider affinity header")
|
||||
alias: Final = (
|
||||
"contract-affinity-anthropic" if case is CLAUDE_SONNET_5_5_TEST_CASE else "contract-affinity-responses"
|
||||
)
|
||||
configured: Final = replace(
|
||||
case,
|
||||
scenario="provider_affinity_header",
|
||||
litellm_request={**case.litellm_request, "model": alias},
|
||||
expected_provider_headers={**case.expected_provider_headers, "x-session-affinity": "contract-session-1"},
|
||||
expected_litellm_response={**case.expected_litellm_response, "model": alias},
|
||||
)
|
||||
body: Final = assert_translation(
|
||||
configured, gateway, provider, headers={"x-litellm-session-id": "contract-session-1"}
|
||||
)
|
||||
messages_structure(body)
|
||||
|
||||
|
||||
def test_messages_metadata_does_not_leak_unsupported_prompt_cache_key_to_vertex_gemma(
|
||||
gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
alias: Final = scenario.model(
|
||||
model="vertex_ai/gemma/gemma-4-26b-a4b-it",
|
||||
api_base=f"{provider.wire.url}/v1/projects/synthetic-vertex-project/locations/global/endpoints/contract:predict",
|
||||
vertex_project="synthetic-vertex-project",
|
||||
vertex_location="global",
|
||||
vertex_credentials=json.dumps(
|
||||
{
|
||||
"type": "external_account",
|
||||
"audience": "synthetic-vertex-audience",
|
||||
"subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
|
||||
"token_url": f"{gateway.upstream_url}/_oauth/token",
|
||||
"credential_source": {"url": f"{gateway.upstream_url}/health"},
|
||||
}
|
||||
),
|
||||
)
|
||||
provider.expect(Reply(body=json.dumps({"predictions": text_completion("gemma-4-26b-a4b-it")}).encode()))
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": alias,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"metadata": {"user_id": "contract-session-1"},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert_provider_request(
|
||||
provider.received(),
|
||||
"/v1/projects/synthetic-vertex-project/locations/global/endpoints/contract:predict",
|
||||
{"content-type": "application/json", "authorization": "Bearer scripted-token"},
|
||||
{
|
||||
"instances": [
|
||||
{
|
||||
"@requestFormat": "chatCompletions",
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"max_tokens": 64,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = JSON_OBJECT.validate_json(response.content)
|
||||
messages_structure(body)
|
||||
assert body == {
|
||||
"id": "chatcmpl-contract-text",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": alias,
|
||||
"content": [{"type": "text", "text": "Hello."}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_details": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 2},
|
||||
}
|
||||
|
|
@ -5,8 +5,16 @@ import pytest
|
|||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE
|
||||
from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE
|
||||
from integration.translation.messages.bases.azure_ai import (
|
||||
CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
)
|
||||
from integration.translation.messages.bases.vertex_ai import (
|
||||
CLAUDE_SONNET_5_5_TEST_CASE as VERTEX_CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
)
|
||||
from integration.translation.response_contracts import messages_structure
|
||||
from integration.translation.runner import assert_translation
|
||||
from pydantic import JsonValue
|
||||
|
||||
SIGNATURE_1: Final = (
|
||||
"EpECCqgBCBIYAipAivUPApu85FYYe3+cXal8EiJOza7QGqKyekC8vDSn4oyeqGa2CrarO4abiuG7dzBXjmYR8+daw4h50ZjKmak7czIRY2xh"
|
||||
|
|
@ -67,6 +75,71 @@ CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE], ids=lambda case: case.id)
|
||||
@pytest.mark.parametrize(
|
||||
"case", [CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE], ids=(CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE.id,)
|
||||
)
|
||||
def test_messages_reasoning_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
(CLAUDE_SONNET_5_5_TEST_CASE, AZURE_CLAUDE_SONNET_5_5_TEST_CASE, VERTEX_CLAUDE_SONNET_5_5_TEST_CASE),
|
||||
ids=(CLAUDE_SONNET_5_5_TEST_CASE.id, AZURE_CLAUDE_SONNET_5_5_TEST_CASE.id, VERTEX_CLAUDE_SONNET_5_5_TEST_CASE.id),
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"thinking,signature,preserve",
|
||||
(
|
||||
pytest.param("I need the file before answering.", "scripted-signed-history", True, id="visible"),
|
||||
pytest.param("", "scripted-signed-history", True, id="signed_empty"),
|
||||
pytest.param("", "", False, id="empty_signature"),
|
||||
pytest.param("", None, False, id="unsigned_empty"),
|
||||
pytest.param("I need the file before answering.", None, True, id="unsigned_visible"),
|
||||
),
|
||||
)
|
||||
def test_messages_replays_signed_thinking_and_tool_results_without_losing_blocks(
|
||||
case: TranslationTestCase,
|
||||
thinking: str,
|
||||
signature: str | None,
|
||||
preserve: bool,
|
||||
gateway: Gateway,
|
||||
provider: SharedProvider,
|
||||
) -> None:
|
||||
content: Final[list[JsonValue]] = [
|
||||
{"type": "thinking", "thinking": thinking, **({"signature": signature} if signature is not None else {})},
|
||||
{"type": "redacted_thinking", "data": "scripted-redacted-history"},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_history",
|
||||
"name": "read_file",
|
||||
"input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200},
|
||||
},
|
||||
]
|
||||
history: Final[list[JsonValue]] = [
|
||||
{"role": "user", "content": "Read the validation file."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": content,
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_history", "content": "quantity must be positive"}
|
||||
],
|
||||
},
|
||||
]
|
||||
replay: Final = replace(
|
||||
case,
|
||||
scenario="signed_thinking_replay",
|
||||
litellm_request={**case.litellm_request, "messages": history},
|
||||
expected_provider_request={
|
||||
**case.expected_provider_request,
|
||||
"messages": [
|
||||
history[0],
|
||||
{"role": "assistant", "content": content if preserve else content[1:]},
|
||||
history[2],
|
||||
],
|
||||
},
|
||||
)
|
||||
body: Final = assert_translation(replay, gateway, provider)
|
||||
messages_structure(body)
|
||||
|
|
|
|||
289
tests/integration/translation/response_contracts.py
Normal file
289
tests/integration/translation/response_contracts.py
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from integration._support.client import object_value
|
||||
from integration.translation.chat_completions.cross_model.contracts import (
|
||||
CacheCreationDetails,
|
||||
Contract,
|
||||
NonemptyString,
|
||||
TokenCount,
|
||||
)
|
||||
from pydantic import Field, JsonValue, TypeAdapter
|
||||
|
||||
JSON_ARGUMENTS: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
class TextBlock(Contract):
|
||||
type: Literal["text"]
|
||||
text: str
|
||||
citations: list[dict[str, JsonValue]] | None = None
|
||||
|
||||
|
||||
class ThinkingBlock(Contract):
|
||||
type: Literal["thinking"]
|
||||
thinking: str
|
||||
signature: NonemptyString
|
||||
|
||||
|
||||
class RedactedThinkingBlock(Contract):
|
||||
type: Literal["redacted_thinking"]
|
||||
data: NonemptyString
|
||||
|
||||
|
||||
class ToolUseBlock(Contract):
|
||||
type: Literal["tool_use"]
|
||||
id: NonemptyString
|
||||
name: NonemptyString
|
||||
input: dict[str, JsonValue]
|
||||
caller: dict[str, JsonValue] | None = None
|
||||
|
||||
|
||||
MessageBlock = Annotated[TextBlock | ThinkingBlock | RedactedThinkingBlock | ToolUseBlock, Field(discriminator="type")]
|
||||
|
||||
|
||||
class MessageOutputDetails(Contract):
|
||||
thinking_tokens: TokenCount | None = None
|
||||
|
||||
|
||||
class MessageServerToolUse(Contract):
|
||||
web_search_requests: TokenCount | None = None
|
||||
web_fetch_requests: TokenCount | None = None
|
||||
|
||||
|
||||
class MessageUsage(Contract):
|
||||
input_tokens: TokenCount
|
||||
output_tokens: TokenCount
|
||||
cache_creation_input_tokens: TokenCount | None = None
|
||||
cache_read_input_tokens: TokenCount | None = None
|
||||
cache_creation: CacheCreationDetails | None = None
|
||||
output_tokens_details: MessageOutputDetails | None = None
|
||||
server_tool_use: MessageServerToolUse | None = None
|
||||
service_tier: str | None = None
|
||||
inference_geo: str | None = None
|
||||
|
||||
|
||||
class MessagesResponse(Contract):
|
||||
id: NonemptyString
|
||||
type: Literal["message"]
|
||||
role: Literal["assistant"]
|
||||
model: NonemptyString
|
||||
content: Annotated[list[MessageBlock], Field(min_length=1)]
|
||||
stop_reason: Literal["end_turn", "tool_use", "max_tokens", "stop_sequence", "pause_turn", "refusal"]
|
||||
stop_sequence: str | None
|
||||
usage: MessageUsage
|
||||
container: dict[str, JsonValue] | None = None
|
||||
diagnostics: dict[str, JsonValue] | None = None
|
||||
stop_details: dict[str, JsonValue] | None = None
|
||||
|
||||
|
||||
class OutputText(Contract):
|
||||
type: Literal["output_text"]
|
||||
text: str
|
||||
annotations: list[dict[str, JsonValue]]
|
||||
logprobs: list[dict[str, JsonValue]] | None = None
|
||||
|
||||
|
||||
class OutputMessage(Contract):
|
||||
type: Literal["message"]
|
||||
id: NonemptyString
|
||||
status: Literal["completed", "incomplete", "in_progress"]
|
||||
role: Literal["assistant"]
|
||||
content: Annotated[list[OutputText], Field(min_length=1)]
|
||||
phase: str | None = None
|
||||
|
||||
|
||||
class ReasoningSummary(Contract):
|
||||
type: Literal["summary_text"]
|
||||
text: str
|
||||
|
||||
|
||||
class OutputReasoning(Contract):
|
||||
type: Literal["reasoning"]
|
||||
id: NonemptyString
|
||||
summary: list[ReasoningSummary] | None = None
|
||||
content: list[OutputText] | None = None
|
||||
encrypted_content: str | None = None
|
||||
|
||||
|
||||
class OutputFunctionCall(Contract):
|
||||
type: Literal["function_call"]
|
||||
id: NonemptyString
|
||||
call_id: NonemptyString
|
||||
name: NonemptyString
|
||||
arguments: str
|
||||
status: Literal["completed", "incomplete", "in_progress"] | None = None
|
||||
namespace: str | None = None
|
||||
|
||||
|
||||
OutputItem = Annotated[OutputMessage | OutputReasoning | OutputFunctionCall, Field(discriminator="type")]
|
||||
|
||||
|
||||
class InputTokenDetails(Contract):
|
||||
cached_tokens: TokenCount | None = None
|
||||
cache_write_tokens: TokenCount | None = None
|
||||
text_tokens: TokenCount | None = None
|
||||
audio_tokens: TokenCount | None = None
|
||||
image_tokens: TokenCount | None = None
|
||||
video_tokens: TokenCount | None = None
|
||||
|
||||
|
||||
class OutputTokenDetails(Contract):
|
||||
reasoning_tokens: TokenCount | None = None
|
||||
text_tokens: TokenCount | None = None
|
||||
audio_tokens: TokenCount | None = None
|
||||
|
||||
|
||||
class ResponsesUsage(Contract):
|
||||
input_tokens: TokenCount
|
||||
output_tokens: TokenCount
|
||||
total_tokens: TokenCount
|
||||
input_tokens_details: InputTokenDetails | None = None
|
||||
output_tokens_details: OutputTokenDetails | None = None
|
||||
|
||||
|
||||
class ResponsesResponse(Contract):
|
||||
id: NonemptyString
|
||||
object: Literal["response"]
|
||||
created_at: Annotated[int | float, Field(ge=0)]
|
||||
status: Literal["completed", "incomplete", "failed", "in_progress", "cancelled", "queued"]
|
||||
error: dict[str, JsonValue] | None
|
||||
incomplete_details: dict[str, JsonValue] | None
|
||||
instructions: str | None
|
||||
metadata: dict[str, str]
|
||||
model: NonemptyString
|
||||
output: Annotated[list[OutputItem], Field(min_length=1)]
|
||||
parallel_tool_calls: bool
|
||||
tool_choice: str | dict[str, JsonValue]
|
||||
tools: list[dict[str, JsonValue]]
|
||||
usage: ResponsesUsage
|
||||
max_output_tokens: TokenCount | None = None
|
||||
previous_response_id: str | None = None
|
||||
reasoning: dict[str, JsonValue] | None = None
|
||||
store: bool | None = None
|
||||
temperature: float | None = None
|
||||
text: dict[str, JsonValue] | None = None
|
||||
top_p: float | None = None
|
||||
truncation: str | None = None
|
||||
user: str | None = None
|
||||
background: bool | None = None
|
||||
completed_at: Annotated[int | float, Field(ge=0)] | None = None
|
||||
frequency_penalty: float | None = None
|
||||
max_tool_calls: TokenCount | None = None
|
||||
presence_penalty: float | None = None
|
||||
prompt_cache_key: str | None = None
|
||||
prompt_cache_retention: str | None = None
|
||||
safety_identifier: str | None = None
|
||||
service_tier: str | None = None
|
||||
top_logprobs: TokenCount | None = None
|
||||
provider_specific_fields: dict[str, JsonValue] | None = None
|
||||
access_programs: dict[str, JsonValue] | None = None
|
||||
billing: dict[str, JsonValue] | None = None
|
||||
moderation: dict[str, JsonValue] | None = None
|
||||
tool_usage: dict[str, JsonValue] | None = None
|
||||
|
||||
|
||||
MESSAGES_OPTIONAL: Final = MappingProxyType(
|
||||
{
|
||||
"": frozenset(("container", "diagnostics", "stop_details")),
|
||||
"content[]": frozenset(("caller", "citations")),
|
||||
"usage": frozenset(
|
||||
(
|
||||
"cache_creation_input_tokens",
|
||||
"cache_read_input_tokens",
|
||||
"cache_creation",
|
||||
"output_tokens_details",
|
||||
"server_tool_use",
|
||||
"service_tier",
|
||||
"inference_geo",
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
RESPONSES_OPTIONAL: Final = MappingProxyType(
|
||||
{
|
||||
"": frozenset(
|
||||
(
|
||||
"parallel_tool_calls",
|
||||
"tool_choice",
|
||||
"tools",
|
||||
"max_output_tokens",
|
||||
"previous_response_id",
|
||||
"reasoning",
|
||||
"store",
|
||||
"temperature",
|
||||
"text",
|
||||
"top_p",
|
||||
"truncation",
|
||||
"user",
|
||||
"background",
|
||||
"completed_at",
|
||||
"frequency_penalty",
|
||||
"max_tool_calls",
|
||||
"presence_penalty",
|
||||
"prompt_cache_key",
|
||||
"prompt_cache_retention",
|
||||
"safety_identifier",
|
||||
"service_tier",
|
||||
"top_logprobs",
|
||||
"provider_specific_fields",
|
||||
"access_programs",
|
||||
"billing",
|
||||
"moderation",
|
||||
"tool_usage",
|
||||
)
|
||||
),
|
||||
"output[]": frozenset(("namespace", "status", "phase")),
|
||||
"output[].content[]": frozenset(("logprobs",)),
|
||||
"usage": frozenset(("input_tokens_details", "output_tokens_details", "cost")),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def messages_structure(body: Mapping[str, JsonValue]) -> JsonValue:
|
||||
response: Final = MessagesResponse.model_validate(body)
|
||||
tools: Final = tuple(block for block in response.content if isinstance(block, ToolUseBlock))
|
||||
assert len({tool.id for tool in tools}) == len(tools), body
|
||||
assert response.stop_reason == ("tool_use" if tools else "end_turn"), body
|
||||
return _structure(dict(body), "", MESSAGES_OPTIONAL, "content", "tool_use")
|
||||
|
||||
|
||||
def responses_structure(body: Mapping[str, JsonValue]) -> JsonValue:
|
||||
response: Final = ResponsesResponse.model_validate(body)
|
||||
assert response.status == "completed", body
|
||||
assert response.error is None and response.incomplete_details is None, body
|
||||
assert response.usage.total_tokens == response.usage.input_tokens + response.usage.output_tokens, body
|
||||
tools: Final = tuple(item for item in response.output if isinstance(item, OutputFunctionCall))
|
||||
assert len({tool.call_id for tool in tools}) == len(tools), body
|
||||
assert len({item.id for item in response.output}) == len(response.output), body
|
||||
for tool in tools:
|
||||
JSON_ARGUMENTS.validate_json(tool.arguments)
|
||||
return _structure(dict(body), "", RESPONSES_OPTIONAL, "output", "function_call")
|
||||
|
||||
|
||||
def _structure(
|
||||
value: JsonValue,
|
||||
path: str,
|
||||
optional: Mapping[str, frozenset[str]],
|
||||
items_path: str,
|
||||
tool_type: str,
|
||||
) -> JsonValue:
|
||||
if isinstance(value, dict):
|
||||
excluded: Final = optional.get(path, frozenset[str]())
|
||||
return {
|
||||
key: _structure(item, f"{path}.{key}" if path else key, optional, items_path, tool_type)
|
||||
for key, item in value.items()
|
||||
if key not in excluded
|
||||
}
|
||||
if isinstance(value, list):
|
||||
tools: Final = tuple(
|
||||
item for item in value if path == items_path and object_value(item).get("type") == tool_type
|
||||
)
|
||||
compared: Final = tools or tuple(value)
|
||||
shapes: Final = tuple(_structure(item, f"{path}[]", optional, items_path, tool_type) for item in compared)
|
||||
ordered: Final = tuple(sorted({json.dumps(shape, sort_keys=True) for shape in shapes}))
|
||||
return [TypeAdapter(JsonValue).validate_json(shape) for shape in ordered]
|
||||
if path == "created_at":
|
||||
return "number"
|
||||
return type(value).__name__
|
||||
80
tests/integration/translation/response_scenarios.py
Normal file
80
tests/integration/translation/response_scenarios.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import object_value
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from pydantic import JsonValue
|
||||
|
||||
COMMENTARY: Final = ("I found the validation entry point. ", "I will read it next.")
|
||||
|
||||
|
||||
def text_completion(model: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"id": "chatcmpl-contract-text",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello."}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
|
||||
}
|
||||
|
||||
|
||||
MESSAGE_ITEMS: Final[tuple[Mapping[str, JsonValue], ...]] = tuple(
|
||||
{
|
||||
"type": "message",
|
||||
"id": f"msg_contract_{index}",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"phase": "commentary",
|
||||
"content": [{"type": "output_text", "text": text, "annotations": [], "logprobs": []}],
|
||||
}
|
||||
for index, text in enumerate(COMMENTARY)
|
||||
)
|
||||
|
||||
|
||||
def items(value: JsonValue) -> tuple[dict[str, JsonValue], ...]:
|
||||
assert isinstance(value, list), value
|
||||
return tuple(object_value(item) for item in value)
|
||||
|
||||
|
||||
def two_messages_before_tools(case: TranslationTestCase) -> TranslationTestCase:
|
||||
provider_tools: Final = tuple(
|
||||
item for item in items(case.mock_provider_response["output"]) if item["type"] == "function_call"
|
||||
)
|
||||
assert provider_tools, case.id
|
||||
return replace(
|
||||
case,
|
||||
scenario="two_messages_before_tools",
|
||||
mock_provider_response={
|
||||
**case.mock_provider_response,
|
||||
"output": [*(dict(item) for item in MESSAGE_ITEMS), *provider_tools],
|
||||
},
|
||||
expected_litellm_response=_messages_then_tools_response(case),
|
||||
)
|
||||
|
||||
|
||||
def _messages_then_tools_response(case: TranslationTestCase) -> Mapping[str, JsonValue]:
|
||||
expected: Final = case.expected_litellm_response
|
||||
if case.litellm_endpoint == "/v1/chat/completions":
|
||||
choice: Final = items(expected["choices"])[0]
|
||||
return {
|
||||
**expected,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "".join(COMMENTARY),
|
||||
"tool_calls": object_value(choice["message"])["tool_calls"],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
if case.litellm_endpoint == "/v1/messages":
|
||||
tools: Final = tuple(item for item in items(expected["content"]) if item["type"] == "tool_use")
|
||||
return {**expected, "content": [*({"type": "text", "text": text} for text in COMMENTARY), *tools]}
|
||||
assert case.litellm_endpoint == "/v1/responses", case.litellm_endpoint
|
||||
calls: Final = tuple(item for item in items(expected["output"]) if item["type"] == "function_call")
|
||||
return {**expected, "output": [*(dict(item) for item in MESSAGE_ITEMS), *calls]}
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import ANY
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy_process
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration._support.responses_vendor import same_response
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.response_contracts import responses_structure
|
||||
from integration.translation.response_scenarios import two_messages_before_tools
|
||||
from integration.translation.responses.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE
|
||||
from integration.translation.responses.bases.azure_ai import (
|
||||
CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
)
|
||||
from integration.translation.responses.bases.openai_responses import GPT_5_6_SOL_TEST_CASE, GPT_6_1_SOL_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE)
|
||||
AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace(
|
||||
AZURE_CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
scenario="cross_model_rich_request",
|
||||
litellm_request={**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, "model": "azure_ai/claude-sonnet-5-5"},
|
||||
expected_provider_request=CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request,
|
||||
mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response,
|
||||
expected_litellm_response={
|
||||
**CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response,
|
||||
"model": "azure_ai/claude-sonnet-5-5",
|
||||
},
|
||||
)
|
||||
GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE: Final = replace(
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
scenario="previous_response_id",
|
||||
litellm_request={**GPT_5_6_SOL_TEST_CASE.litellm_request, "model": "openai/responses/gpt-6.1-sol"},
|
||||
expected_provider_request={**GPT_5_6_SOL_TEST_CASE.expected_provider_request, "model": "gpt-6.1-sol"},
|
||||
mock_provider_response={**GPT_5_6_SOL_TEST_CASE.mock_provider_response, "model": "gpt-6.1-sol"},
|
||||
expected_litellm_response={
|
||||
**GPT_5_6_SOL_TEST_CASE.expected_litellm_response,
|
||||
"model": GPT_6_1_SOL_TEST_CASE.expected_litellm_response["model"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
(CLAUDE_SONNET_5_5_TEST_CASE, AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE),
|
||||
ids=(CLAUDE_SONNET_5_5_TEST_CASE.id, AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE.id),
|
||||
)
|
||||
def test_responses_keep_the_same_public_contract_on_native_and_bridged_routes(
|
||||
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert {key: value for key, value in case.litellm_request.items() if key != "model"} == {
|
||||
key: value for key, value in GPT_6_1_SOL_TEST_CASE.litellm_request.items() if key != "model"
|
||||
}
|
||||
native: Final = assert_translation(GPT_6_1_SOL_TEST_CASE, gateway, provider)
|
||||
compared: Final = assert_translation(case, gateway, provider)
|
||||
assert responses_structure(compared) == responses_structure(native)
|
||||
|
||||
|
||||
def test_responses_preserves_multiple_message_items_before_tools(gateway: Gateway, provider: SharedProvider) -> None:
|
||||
body: Final = assert_translation(GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE, gateway, provider)
|
||||
responses_structure(body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
(GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE, AZURE_CLAUDE_SONNET_5_5_TEST_CASE),
|
||||
ids=(GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE.id, AZURE_CLAUDE_SONNET_5_5_TEST_CASE.id),
|
||||
)
|
||||
def test_previous_response_id_replays_the_turn_without_leaking_internal_routing_fields(
|
||||
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider, tmp_path: Path
|
||||
) -> None:
|
||||
stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
config: Final = tmp_path / "stored-responses.yaml"
|
||||
config.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**stock,
|
||||
"general_settings": {**object_value(stock["general_settings"]), "store_prompts_in_spend_logs": True},
|
||||
}
|
||||
)
|
||||
)
|
||||
with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned:
|
||||
assert_previous_response_replay(case, owned.gateway, provider)
|
||||
|
||||
|
||||
def assert_previous_response_replay(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
identity: Final = f"resp_contract_{uuid4().hex}"
|
||||
first_case: Final = replace(
|
||||
case,
|
||||
mock_provider_response={**case.mock_provider_response, "id": identity},
|
||||
expected_litellm_response={**case.expected_litellm_response, "id": ANY},
|
||||
)
|
||||
first: Final = assert_translation(first_case, gateway, provider)
|
||||
responses_structure(first)
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
|
||||
(string_value(case.litellm_request["model"]),),
|
||||
),
|
||||
lambda rows: any(same_response(str(row["request_id"]), str(first["id"])) for row in rows),
|
||||
seconds=70,
|
||||
)
|
||||
previous_identifier: Final = next(
|
||||
str(row["request_id"]) for row in rows if same_response(str(row["request_id"]), str(first["id"]))
|
||||
)
|
||||
expected_request: Final = (
|
||||
{**case.expected_provider_request, "input": "Say hello again.", "previous_response_id": identity}
|
||||
if case is GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE
|
||||
else {
|
||||
**case.expected_provider_request,
|
||||
"messages": [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Say hello."}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Hello."}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "Say hello again."}]},
|
||||
],
|
||||
}
|
||||
)
|
||||
followup: Final = replace(
|
||||
first_case,
|
||||
litellm_request={**case.litellm_request, "input": "Say hello again.", "previous_response_id": first["id"]},
|
||||
expected_provider_request=expected_request,
|
||||
mock_provider_response={**case.mock_provider_response, "id": f"{identity}_second"},
|
||||
expected_litellm_response={
|
||||
**case.expected_litellm_response,
|
||||
"id": ANY,
|
||||
"previous_response_id": None if case is GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE else previous_identifier,
|
||||
},
|
||||
)
|
||||
responses_structure(assert_translation(followup, gateway, provider))
|
||||
|
|
@ -34,9 +34,15 @@ def expected_response_for_deployment(
|
|||
}
|
||||
|
||||
|
||||
def assert_translation(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> dict[str, JsonValue]:
|
||||
def assert_translation(
|
||||
case: TranslationTestCase,
|
||||
gateway: Gateway,
|
||||
provider: SharedProvider,
|
||||
*,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
) -> dict[str, JsonValue]:
|
||||
provider.expect(Reply(status=case.mock_provider_status_code, body=json.dumps(case.mock_provider_response).encode()))
|
||||
response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request)
|
||||
response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request, headers=headers)
|
||||
received: Final = provider.received()
|
||||
assert_provider_request(
|
||||
received,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue