From 4496fe0ebcc0193a457137acfd9302dfa5abba3e Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 15 Oct 2026 00:05:09 -0700 Subject: [PATCH] test: cover response contracts across translation endpoints (#46632) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../providers/test_databricks_chat_wire.py | 40 +++ ...test_hosted_vllm_reasoning_content_wire.py | 2 + tests/integration/proxy_config.yaml | 12 + .../cross_model/test_response_scenarios.py | 237 ++++++++++++++ .../cross_model/test_messages_cross_model.py | 143 +++++++++ .../test_messages_reasoning_anthropic.py | 77 ++++- .../translation/response_contracts.py | 289 ++++++++++++++++++ .../translation/response_scenarios.py | 80 +++++ .../cross_model/test_responses_cross_model.py | 135 ++++++++ tests/integration/translation/runner.py | 10 +- 10 files changed, 1021 insertions(+), 4 deletions(-) create mode 100644 tests/integration/translation/chat_completions/cross_model/test_response_scenarios.py create mode 100644 tests/integration/translation/messages/cross_model/test_messages_cross_model.py create mode 100644 tests/integration/translation/response_contracts.py create mode 100644 tests/integration/translation/response_scenarios.py create mode 100644 tests/integration/translation/responses/cross_model/test_responses_cross_model.py diff --git a/tests/integration/providers/test_databricks_chat_wire.py b/tests/integration/providers/test_databricks_chat_wire.py index 9681d0cd66c..016366c040b 100644 --- a/tests/integration/providers/test_databricks_chat_wire.py +++ b/tests/integration/providers/test_databricks_chat_wire.py @@ -13,6 +13,8 @@ from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.process import owned_proxy_process from integration._support.wire import Reply, Request, wire_server +from integration.translation.chat_completions.cross_model.contracts import assert_completion +from integration.translation.response_contracts import responses_structure from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter _BACKEND: Final = "databricks-glm-5-2" @@ -81,6 +83,43 @@ _JSON_TOOL_CALL_MESSAGE: Final[dict[str, JsonValue]] = { } +def test_databricks_responses_reasoning_summary_keeps_reasoning_effort_a_string(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/chat/completions" + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _BACKEND, + "messages": [{"role": "user", "content": "Say hello."}], + "max_tokens": 64, + "reasoning_effort": "low", + } + return Reply(body=_chat_completion_response(_BACKEND, {"role": "assistant", "content": "Hello."}, "stop")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + alias: Final = scenario.model(model=f"databricks/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/responses", + { + "model": alias, + "input": "Say hello.", + "max_output_tokens": 64, + "reasoning": {"effort": "low", "summary": "auto"}, + }, + ) + assert response.status_code == 200, response.text + body: Final = _JSON_OBJECT.validate_json(response.content) + responses_structure(body) + assert body["usage"] == { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + "input_tokens_details": None, + "output_tokens_details": None, + "cost": None, + } + assert len(wire.drain()) == 1 + + def _chat_completion_response(model: str, message: dict[str, JsonValue], finish_reason: str) -> bytes: response: Final = { "id": "databricks-json-schema-response", @@ -455,6 +494,7 @@ def test_databricks_json_schema_refs_are_preserved_in_chat_completions(gateway: ) assert response.status_code == 200, response.text body: Final = _JSON_OBJECT.validate_json(response.content) + assert_completion(body) assert body["choices"] == [ { "index": 0, diff --git a/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py b/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py index 0b9f24f538a..60822321e5b 100644 --- a/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py +++ b/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py @@ -9,6 +9,7 @@ import pytest from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, Wire, wire_server +from integration.translation.chat_completions.cross_model.contracts import assert_completion from pydantic import JsonValue, TypeAdapter _BACKEND: Final = "qwen3-reasoning" @@ -186,6 +187,7 @@ def test_hosted_vllm_assistant_reasoning_content_reaches_the_wire(gateway: Gatew ) assert response.status_code == 200, response.text payload: Final = _JSON_OBJECT.validate_json(response.content) + assert_completion(payload) assert payload["id"] == identity _only_request(wire) diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 451764eeef9..f047d6e0b3e 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -336,6 +336,18 @@ model_list: model: openai/responses/gpt-6.1-sol api_base: http://127.0.0.1:8191 api_key: synthetic-openai-key + - model_name: contract-affinity-responses + litellm_params: + model: openai/responses/gpt-6.1-sol + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key + provider_affinity_header: x-session-affinity + - model_name: contract-affinity-anthropic + litellm_params: + model: anthropic/claude-sonnet-5-5 + api_base: http://127.0.0.1:8191 + api_key: synthetic-anthropic-key + provider_affinity_header: x-session-affinity - model_name: azure/gpt-5.4 litellm_params: model: azure/gpt-5.4 diff --git a/tests/integration/translation/chat_completions/cross_model/test_response_scenarios.py b/tests/integration/translation/chat_completions/cross_model/test_response_scenarios.py new file mode 100644 index 00000000000..2a00ac9aa13 --- /dev/null +++ b/tests/integration/translation/chat_completions/cross_model/test_response_scenarios.py @@ -0,0 +1,237 @@ +import json +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, object_value +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from integration.translation.case import SSEFrame, StreamingTranslationTestCase +from integration.translation.chat_completions.bases.openai_responses import GPT_6_1_SOL_TEST_CASE +from integration.translation.chat_completions.cross_model.contracts import ( + Message, + assert_completion, + assert_final_text, + assert_stream_preserved, +) +from integration.translation.response_scenarios import text_completion, two_messages_before_tools +from integration.translation.runner import ( + assert_provider_request, + assert_translation, + parse_sse_frames, + streaming_reply, +) +from openai.lib.streaming.chat import ChatCompletionStreamState +from openai.types.chat import ChatCompletionChunk +from pydantic import JsonValue, TypeAdapter + +OBJECTS: Final = TypeAdapter(list[dict[str, JsonValue]]) + +GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE) +GPT_6_1_SOL_STRING_TOOL_OUTPUT_TEST_CASE: Final = replace( + GPT_6_1_SOL_TEST_CASE, + scenario="plain_string_tool_output", + litellm_request={ + **GPT_6_1_SOL_TEST_CASE.litellm_request, + "messages": [ + {"role": "user", "content": "Read the validation file."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_contract_1", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path":"shop/validation.py"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_contract_1", "content": '{"result":"quantity must be positive"}'}, + ], + }, + expected_provider_request={ + **{ + key: value + for key, value in GPT_6_1_SOL_TEST_CASE.expected_provider_request.items() + if key != "instructions" + }, + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "Read the validation file."}]}, + { + "type": "function_call", + "call_id": "call_contract_1", + "name": "read_file", + "arguments": '{"path":"shop/validation.py"}', + }, + { + "type": "function_call_output", + "call_id": "call_contract_1", + "output": '{"result":"quantity must be positive"}', + }, + ], + }, +) + + +def test_multiple_responses_messages_and_tools_produce_one_complete_choice( + gateway: Gateway, provider: SharedProvider +) -> None: + body: Final = assert_translation(GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE, gateway, provider) + assert_completion(body) + + +def test_chat_to_responses_keeps_plain_string_tool_outputs_as_strings( + gateway: Gateway, provider: SharedProvider +) -> None: + pytest.skip("BUG: LIT-9038 plain-string tool outputs are wrapped in a list that strict Responses backends reject") + body: Final = assert_translation(GPT_6_1_SOL_STRING_TOOL_OUTPUT_TEST_CASE, gateway, provider) + assert_completion(body) + + +def test_azure_deployment_capability_override_filters_temperature_even_with_base_model( + gateway: Gateway, provider: SharedProvider +) -> None: + with gateway.scenario() as scenario: + alias: Final = scenario.model( + model="azure/contract-deployment", + api_base=provider.wire.url, + api_key="synthetic-azure-key", + api_version="2025-04-01-preview", + base_model="azure/gpt-6.1-sol", + drop_params=True, + model_info={"supports_none_reasoning_effort": False}, + ) + provider.expect(Reply(body=json.dumps(text_completion("gpt-6.1-sol")).encode())) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": alias, + "messages": [{"role": "user", "content": "Say hello."}], + "max_tokens": 64, + "temperature": 0.2, + "use_chat_completions_api": True, + }, + ) + assert_provider_request( + provider.received(), + "/openai/deployments/contract-deployment/chat/completions?api-version=2025-04-01-preview", + { + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + { + "model": "contract-deployment", + "messages": [{"role": "user", "content": "Say hello."}], + "max_completion_tokens": 64, + "stream": False, + }, + ) + assert response.status_code == 200, response.text + assert response.json() == { + **text_completion(alias), + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello."}, + "finish_reason": "stop", + "provider_specific_fields": {}, + } + ], + } + assert_final_text(JSON_OBJECT.validate_json(response.content)) + + +def _cached_stream(missing_index: bool, terminal_choices: bool) -> StreamingTranslationTestCase: + index: Final[dict[str, JsonValue]] = {} if missing_index else {"index": 0} + envelope: Final[dict[str, JsonValue]] = { + "id": "chatcmpl-cached-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-6.1-sol", + } + usage: Final[dict[str, JsonValue]] = { + "prompt_tokens": 1000, + "completion_tokens": 100, + "total_tokens": 1100, + "prompt_tokens_details": {"cached_tokens": 600}, + } + finish: Final[dict[str, JsonValue]] = {**index, "delta": {}, "finish_reason": "stop"} + return StreamingTranslationTestCase( + scenario="choice_bearing_cached_usage" if terminal_choices else "usage_only_cached_usage", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "openai/gpt-6.1-sol", + "messages": [{"role": "user", "content": "Say hello."}], + "max_tokens": 64, + "stream": True, + "stream_options": {"include_usage": True}, + "use_chat_completions_api": True, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/chat/completions", + expected_provider_headers=GPT_6_1_SOL_TEST_CASE.expected_provider_headers, + expected_provider_request={ + "model": "gpt-6.1-sol", + "messages": [{"role": "user", "content": "Say hello."}], + "max_completion_tokens": 64, + "stream": True, + "stream_options": {"include_usage": True}, + }, + mock_provider_frames=( + SSEFrame( + data={ + **envelope, + "choices": [{**index, "delta": {"role": "assistant", "content": "Hello."}, "finish_reason": None}], + } + ), + *(() if terminal_choices else (SSEFrame(data={**envelope, "choices": [finish]}),)), + SSEFrame(data={**envelope, "choices": [finish] if terminal_choices else [], "usage": usage}), + SSEFrame(data="[DONE]"), + ), + expected_litellm_message={"role": "assistant", "content": "Hello."}, + expected_litellm_usage={ + "prompt_tokens": 1000, + "completion_tokens": 100, + "total_tokens": 1100, + "prompt_tokens_details": {"cached_tokens": 600}, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 100}, + }, + ) + + +@pytest.mark.parametrize("missing_index", (False, True), ids=("indexed", "omitted_index")) +@pytest.mark.parametrize("terminal_choices", (False, True), ids=("usage_only", "choices_and_usage")) +def test_cached_usage_survives_raw_streaming_and_sdk_assembly( + missing_index: bool, terminal_choices: bool, gateway: Gateway, provider: SharedProvider +) -> None: + if missing_index: + pytest.skip("BUG: LIT-9468 omitted upstream choice indexes reach the SDK as null and abort the stream") + case: Final = _cached_stream(missing_index, terminal_choices) + provider.expect(streaming_reply(case)) + response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request) + assert_provider_request( + provider.received(), + case.expected_provider_endpoint, + case.expected_provider_headers, + case.expected_provider_request, + ) + assert response.status_code == 200, response.text + frames: Final = parse_sse_frames(response.text) + assert frames and tuple(frame.data == "[DONE]" for frame in frames) == (False,) * (len(frames) - 1) + (True,) + chunks: Final = tuple(JSON_OBJECT.validate_python(frame.data) for frame in frames[:-1]) + state: Final = ChatCompletionStreamState() + for chunk in chunks: + assert chunk["object"] == "chat.completion.chunk", chunk + for choice in OBJECTS.validate_python(chunk["choices"]): + assert type(choice["index"]) is int and choice["index"] == 0, choice + state.handle_chunk(ChatCompletionChunk.model_validate(chunk)) + body: Final = JSON_OBJECT.validate_json(state.get_final_completion().model_dump_json()) + assert_final_text(body) + message: Final = Message.model_validate(object_value(OBJECTS.validate_python(body["choices"])[0]["message"])) + usage: Final = object_value(body["usage"]) + assert_stream_preserved(message, usage, case) + raw_usage: Final = tuple(chunk["usage"] for chunk in chunks if chunk.get("usage")) + assert len(raw_usage) == 1, response.text + assert_stream_preserved(message, object_value(raw_usage[0]), case) diff --git a/tests/integration/translation/messages/cross_model/test_messages_cross_model.py b/tests/integration/translation/messages/cross_model/test_messages_cross_model.py new file mode 100644 index 00000000000..ae6fe70fe7c --- /dev/null +++ b/tests/integration/translation/messages/cross_model/test_messages_cross_model.py @@ -0,0 +1,143 @@ +import json +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.messages.bases.azure_ai import ( + CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE, +) +from integration.translation.messages.bases.openai_responses import GPT_6_1_SOL_TEST_CASE +from integration.translation.response_contracts import messages_structure +from integration.translation.response_scenarios import text_completion, two_messages_before_tools +from integration.translation.runner import assert_provider_request, assert_translation + +GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE) +AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( + AZURE_CLAUDE_SONNET_5_5_TEST_CASE, + scenario="cross_model_rich_request", + litellm_request={**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, "model": "azure_ai/claude-sonnet-5-5"}, + expected_provider_request={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, + "system": [{"type": "text", "text": CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request["system"]}], + }, + mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + expected_litellm_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, + "model": "azure_ai/claude-sonnet-5-5", + }, +) + + +@pytest.mark.parametrize( + "case", + (AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE, GPT_6_1_SOL_TEST_CASE), + ids=(AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE.id, GPT_6_1_SOL_TEST_CASE.id), +) +def test_messages_keep_the_same_public_contract_on_native_and_bridged_routes( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert {key: value for key, value in case.litellm_request.items() if key != "model"} == { + key: value for key, value in CLAUDE_SONNET_5_5_TEST_CASE.litellm_request.items() if key != "model" + } + native: Final = assert_translation(CLAUDE_SONNET_5_5_TEST_CASE, gateway, provider) + compared: Final = assert_translation(case, gateway, provider) + assert messages_structure(compared) == messages_structure(native) + + +def test_messages_preserves_multiple_responses_messages_before_tools( + gateway: Gateway, provider: SharedProvider +) -> None: + body: Final = assert_translation(GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE, gateway, provider) + messages_structure(body) + + +@pytest.mark.parametrize( + "case", + (CLAUDE_SONNET_5_5_TEST_CASE, GPT_6_1_SOL_TEST_CASE), + ids=(CLAUDE_SONNET_5_5_TEST_CASE.id, GPT_6_1_SOL_TEST_CASE.id), +) +def test_messages_forwards_the_configured_provider_affinity_header_on_native_and_bridged_routes( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + if case is CLAUDE_SONNET_5_5_TEST_CASE: + pytest.skip("BUG: LIT-8405 native Messages drops the configured provider affinity header") + alias: Final = ( + "contract-affinity-anthropic" if case is CLAUDE_SONNET_5_5_TEST_CASE else "contract-affinity-responses" + ) + configured: Final = replace( + case, + scenario="provider_affinity_header", + litellm_request={**case.litellm_request, "model": alias}, + expected_provider_headers={**case.expected_provider_headers, "x-session-affinity": "contract-session-1"}, + expected_litellm_response={**case.expected_litellm_response, "model": alias}, + ) + body: Final = assert_translation( + configured, gateway, provider, headers={"x-litellm-session-id": "contract-session-1"} + ) + messages_structure(body) + + +def test_messages_metadata_does_not_leak_unsupported_prompt_cache_key_to_vertex_gemma( + gateway: Gateway, provider: SharedProvider +) -> None: + with gateway.scenario() as scenario: + alias: Final = scenario.model( + model="vertex_ai/gemma/gemma-4-26b-a4b-it", + api_base=f"{provider.wire.url}/v1/projects/synthetic-vertex-project/locations/global/endpoints/contract:predict", + vertex_project="synthetic-vertex-project", + vertex_location="global", + vertex_credentials=json.dumps( + { + "type": "external_account", + "audience": "synthetic-vertex-audience", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": f"{gateway.upstream_url}/_oauth/token", + "credential_source": {"url": f"{gateway.upstream_url}/health"}, + } + ), + ) + provider.expect(Reply(body=json.dumps({"predictions": text_completion("gemma-4-26b-a4b-it")}).encode())) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": alias, + "max_tokens": 64, + "messages": [{"role": "user", "content": "Say hello."}], + "metadata": {"user_id": "contract-session-1"}, + }, + ) + assert response.status_code == 200, response.text + assert_provider_request( + provider.received(), + "/v1/projects/synthetic-vertex-project/locations/global/endpoints/contract:predict", + {"content-type": "application/json", "authorization": "Bearer scripted-token"}, + { + "instances": [ + { + "@requestFormat": "chatCompletions", + "messages": [{"role": "user", "content": "Say hello."}], + "max_tokens": 64, + } + ] + }, + ) + assert response.status_code == 200, response.text + body: Final = JSON_OBJECT.validate_json(response.content) + messages_structure(body) + assert body == { + "id": "chatcmpl-contract-text", + "type": "message", + "role": "assistant", + "model": alias, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 2}, + } diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py index 83bf2a36db8..35a0d5f087d 100644 --- a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py @@ -5,8 +5,16 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.messages.bases.azure_ai import ( + CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE, +) +from integration.translation.messages.bases.vertex_ai import ( + CLAUDE_SONNET_5_5_TEST_CASE as VERTEX_CLAUDE_SONNET_5_5_TEST_CASE, +) +from integration.translation.response_contracts import messages_structure from integration.translation.runner import assert_translation +from pydantic import JsonValue SIGNATURE_1: Final = ( "EpECCqgBCBIYAipAivUPApu85FYYe3+cXal8EiJOza7QGqKyekC8vDSn4oyeqGa2CrarO4abiuG7dzBXjmYR8+daw4h50ZjKmak7czIRY2xh" @@ -67,6 +75,71 @@ CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace( ) -@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize( + "case", [CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE], ids=(CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE.id,) +) def test_messages_reasoning_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) + + +@pytest.mark.parametrize( + "case", + (CLAUDE_SONNET_5_5_TEST_CASE, AZURE_CLAUDE_SONNET_5_5_TEST_CASE, VERTEX_CLAUDE_SONNET_5_5_TEST_CASE), + ids=(CLAUDE_SONNET_5_5_TEST_CASE.id, AZURE_CLAUDE_SONNET_5_5_TEST_CASE.id, VERTEX_CLAUDE_SONNET_5_5_TEST_CASE.id), +) +@pytest.mark.parametrize( + "thinking,signature,preserve", + ( + pytest.param("I need the file before answering.", "scripted-signed-history", True, id="visible"), + pytest.param("", "scripted-signed-history", True, id="signed_empty"), + pytest.param("", "", False, id="empty_signature"), + pytest.param("", None, False, id="unsigned_empty"), + pytest.param("I need the file before answering.", None, True, id="unsigned_visible"), + ), +) +def test_messages_replays_signed_thinking_and_tool_results_without_losing_blocks( + case: TranslationTestCase, + thinking: str, + signature: str | None, + preserve: bool, + gateway: Gateway, + provider: SharedProvider, +) -> None: + content: Final[list[JsonValue]] = [ + {"type": "thinking", "thinking": thinking, **({"signature": signature} if signature is not None else {})}, + {"type": "redacted_thinking", "data": "scripted-redacted-history"}, + { + "type": "tool_use", + "id": "toolu_history", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ] + history: Final[list[JsonValue]] = [ + {"role": "user", "content": "Read the validation file."}, + { + "role": "assistant", + "content": content, + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_history", "content": "quantity must be positive"} + ], + }, + ] + replay: Final = replace( + case, + scenario="signed_thinking_replay", + litellm_request={**case.litellm_request, "messages": history}, + expected_provider_request={ + **case.expected_provider_request, + "messages": [ + history[0], + {"role": "assistant", "content": content if preserve else content[1:]}, + history[2], + ], + }, + ) + body: Final = assert_translation(replay, gateway, provider) + messages_structure(body) diff --git a/tests/integration/translation/response_contracts.py b/tests/integration/translation/response_contracts.py new file mode 100644 index 00000000000..c3182d269ff --- /dev/null +++ b/tests/integration/translation/response_contracts.py @@ -0,0 +1,289 @@ +import json +from collections.abc import Mapping +from types import MappingProxyType +from typing import Annotated, Final, Literal + +from integration._support.client import object_value +from integration.translation.chat_completions.cross_model.contracts import ( + CacheCreationDetails, + Contract, + NonemptyString, + TokenCount, +) +from pydantic import Field, JsonValue, TypeAdapter + +JSON_ARGUMENTS: Final = TypeAdapter(dict[str, JsonValue]) + + +class TextBlock(Contract): + type: Literal["text"] + text: str + citations: list[dict[str, JsonValue]] | None = None + + +class ThinkingBlock(Contract): + type: Literal["thinking"] + thinking: str + signature: NonemptyString + + +class RedactedThinkingBlock(Contract): + type: Literal["redacted_thinking"] + data: NonemptyString + + +class ToolUseBlock(Contract): + type: Literal["tool_use"] + id: NonemptyString + name: NonemptyString + input: dict[str, JsonValue] + caller: dict[str, JsonValue] | None = None + + +MessageBlock = Annotated[TextBlock | ThinkingBlock | RedactedThinkingBlock | ToolUseBlock, Field(discriminator="type")] + + +class MessageOutputDetails(Contract): + thinking_tokens: TokenCount | None = None + + +class MessageServerToolUse(Contract): + web_search_requests: TokenCount | None = None + web_fetch_requests: TokenCount | None = None + + +class MessageUsage(Contract): + input_tokens: TokenCount + output_tokens: TokenCount + cache_creation_input_tokens: TokenCount | None = None + cache_read_input_tokens: TokenCount | None = None + cache_creation: CacheCreationDetails | None = None + output_tokens_details: MessageOutputDetails | None = None + server_tool_use: MessageServerToolUse | None = None + service_tier: str | None = None + inference_geo: str | None = None + + +class MessagesResponse(Contract): + id: NonemptyString + type: Literal["message"] + role: Literal["assistant"] + model: NonemptyString + content: Annotated[list[MessageBlock], Field(min_length=1)] + stop_reason: Literal["end_turn", "tool_use", "max_tokens", "stop_sequence", "pause_turn", "refusal"] + stop_sequence: str | None + usage: MessageUsage + container: dict[str, JsonValue] | None = None + diagnostics: dict[str, JsonValue] | None = None + stop_details: dict[str, JsonValue] | None = None + + +class OutputText(Contract): + type: Literal["output_text"] + text: str + annotations: list[dict[str, JsonValue]] + logprobs: list[dict[str, JsonValue]] | None = None + + +class OutputMessage(Contract): + type: Literal["message"] + id: NonemptyString + status: Literal["completed", "incomplete", "in_progress"] + role: Literal["assistant"] + content: Annotated[list[OutputText], Field(min_length=1)] + phase: str | None = None + + +class ReasoningSummary(Contract): + type: Literal["summary_text"] + text: str + + +class OutputReasoning(Contract): + type: Literal["reasoning"] + id: NonemptyString + summary: list[ReasoningSummary] | None = None + content: list[OutputText] | None = None + encrypted_content: str | None = None + + +class OutputFunctionCall(Contract): + type: Literal["function_call"] + id: NonemptyString + call_id: NonemptyString + name: NonemptyString + arguments: str + status: Literal["completed", "incomplete", "in_progress"] | None = None + namespace: str | None = None + + +OutputItem = Annotated[OutputMessage | OutputReasoning | OutputFunctionCall, Field(discriminator="type")] + + +class InputTokenDetails(Contract): + cached_tokens: TokenCount | None = None + cache_write_tokens: TokenCount | None = None + text_tokens: TokenCount | None = None + audio_tokens: TokenCount | None = None + image_tokens: TokenCount | None = None + video_tokens: TokenCount | None = None + + +class OutputTokenDetails(Contract): + reasoning_tokens: TokenCount | None = None + text_tokens: TokenCount | None = None + audio_tokens: TokenCount | None = None + + +class ResponsesUsage(Contract): + input_tokens: TokenCount + output_tokens: TokenCount + total_tokens: TokenCount + input_tokens_details: InputTokenDetails | None = None + output_tokens_details: OutputTokenDetails | None = None + + +class ResponsesResponse(Contract): + id: NonemptyString + object: Literal["response"] + created_at: Annotated[int | float, Field(ge=0)] + status: Literal["completed", "incomplete", "failed", "in_progress", "cancelled", "queued"] + error: dict[str, JsonValue] | None + incomplete_details: dict[str, JsonValue] | None + instructions: str | None + metadata: dict[str, str] + model: NonemptyString + output: Annotated[list[OutputItem], Field(min_length=1)] + parallel_tool_calls: bool + tool_choice: str | dict[str, JsonValue] + tools: list[dict[str, JsonValue]] + usage: ResponsesUsage + max_output_tokens: TokenCount | None = None + previous_response_id: str | None = None + reasoning: dict[str, JsonValue] | None = None + store: bool | None = None + temperature: float | None = None + text: dict[str, JsonValue] | None = None + top_p: float | None = None + truncation: str | None = None + user: str | None = None + background: bool | None = None + completed_at: Annotated[int | float, Field(ge=0)] | None = None + frequency_penalty: float | None = None + max_tool_calls: TokenCount | None = None + presence_penalty: float | None = None + prompt_cache_key: str | None = None + prompt_cache_retention: str | None = None + safety_identifier: str | None = None + service_tier: str | None = None + top_logprobs: TokenCount | None = None + provider_specific_fields: dict[str, JsonValue] | None = None + access_programs: dict[str, JsonValue] | None = None + billing: dict[str, JsonValue] | None = None + moderation: dict[str, JsonValue] | None = None + tool_usage: dict[str, JsonValue] | None = None + + +MESSAGES_OPTIONAL: Final = MappingProxyType( + { + "": frozenset(("container", "diagnostics", "stop_details")), + "content[]": frozenset(("caller", "citations")), + "usage": frozenset( + ( + "cache_creation_input_tokens", + "cache_read_input_tokens", + "cache_creation", + "output_tokens_details", + "server_tool_use", + "service_tier", + "inference_geo", + ) + ), + } +) +RESPONSES_OPTIONAL: Final = MappingProxyType( + { + "": frozenset( + ( + "parallel_tool_calls", + "tool_choice", + "tools", + "max_output_tokens", + "previous_response_id", + "reasoning", + "store", + "temperature", + "text", + "top_p", + "truncation", + "user", + "background", + "completed_at", + "frequency_penalty", + "max_tool_calls", + "presence_penalty", + "prompt_cache_key", + "prompt_cache_retention", + "safety_identifier", + "service_tier", + "top_logprobs", + "provider_specific_fields", + "access_programs", + "billing", + "moderation", + "tool_usage", + ) + ), + "output[]": frozenset(("namespace", "status", "phase")), + "output[].content[]": frozenset(("logprobs",)), + "usage": frozenset(("input_tokens_details", "output_tokens_details", "cost")), + } +) + + +def messages_structure(body: Mapping[str, JsonValue]) -> JsonValue: + response: Final = MessagesResponse.model_validate(body) + tools: Final = tuple(block for block in response.content if isinstance(block, ToolUseBlock)) + assert len({tool.id for tool in tools}) == len(tools), body + assert response.stop_reason == ("tool_use" if tools else "end_turn"), body + return _structure(dict(body), "", MESSAGES_OPTIONAL, "content", "tool_use") + + +def responses_structure(body: Mapping[str, JsonValue]) -> JsonValue: + response: Final = ResponsesResponse.model_validate(body) + assert response.status == "completed", body + assert response.error is None and response.incomplete_details is None, body + assert response.usage.total_tokens == response.usage.input_tokens + response.usage.output_tokens, body + tools: Final = tuple(item for item in response.output if isinstance(item, OutputFunctionCall)) + assert len({tool.call_id for tool in tools}) == len(tools), body + assert len({item.id for item in response.output}) == len(response.output), body + for tool in tools: + JSON_ARGUMENTS.validate_json(tool.arguments) + return _structure(dict(body), "", RESPONSES_OPTIONAL, "output", "function_call") + + +def _structure( + value: JsonValue, + path: str, + optional: Mapping[str, frozenset[str]], + items_path: str, + tool_type: str, +) -> JsonValue: + if isinstance(value, dict): + excluded: Final = optional.get(path, frozenset[str]()) + return { + key: _structure(item, f"{path}.{key}" if path else key, optional, items_path, tool_type) + for key, item in value.items() + if key not in excluded + } + if isinstance(value, list): + tools: Final = tuple( + item for item in value if path == items_path and object_value(item).get("type") == tool_type + ) + compared: Final = tools or tuple(value) + shapes: Final = tuple(_structure(item, f"{path}[]", optional, items_path, tool_type) for item in compared) + ordered: Final = tuple(sorted({json.dumps(shape, sort_keys=True) for shape in shapes})) + return [TypeAdapter(JsonValue).validate_json(shape) for shape in ordered] + if path == "created_at": + return "number" + return type(value).__name__ diff --git a/tests/integration/translation/response_scenarios.py b/tests/integration/translation/response_scenarios.py new file mode 100644 index 00000000000..b49d2b71441 --- /dev/null +++ b/tests/integration/translation/response_scenarios.py @@ -0,0 +1,80 @@ +from collections.abc import Mapping +from dataclasses import replace +from typing import Final + +from integration._support.client import object_value +from integration.translation.case import TranslationTestCase +from pydantic import JsonValue + +COMMENTARY: Final = ("I found the validation entry point. ", "I will read it next.") + + +def text_completion(model: str) -> dict[str, JsonValue]: + return { + "id": "chatcmpl-contract-text", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello."}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, + } + + +MESSAGE_ITEMS: Final[tuple[Mapping[str, JsonValue], ...]] = tuple( + { + "type": "message", + "id": f"msg_contract_{index}", + "status": "completed", + "role": "assistant", + "phase": "commentary", + "content": [{"type": "output_text", "text": text, "annotations": [], "logprobs": []}], + } + for index, text in enumerate(COMMENTARY) +) + + +def items(value: JsonValue) -> tuple[dict[str, JsonValue], ...]: + assert isinstance(value, list), value + return tuple(object_value(item) for item in value) + + +def two_messages_before_tools(case: TranslationTestCase) -> TranslationTestCase: + provider_tools: Final = tuple( + item for item in items(case.mock_provider_response["output"]) if item["type"] == "function_call" + ) + assert provider_tools, case.id + return replace( + case, + scenario="two_messages_before_tools", + mock_provider_response={ + **case.mock_provider_response, + "output": [*(dict(item) for item in MESSAGE_ITEMS), *provider_tools], + }, + expected_litellm_response=_messages_then_tools_response(case), + ) + + +def _messages_then_tools_response(case: TranslationTestCase) -> Mapping[str, JsonValue]: + expected: Final = case.expected_litellm_response + if case.litellm_endpoint == "/v1/chat/completions": + choice: Final = items(expected["choices"])[0] + return { + **expected, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "".join(COMMENTARY), + "tool_calls": object_value(choice["message"])["tool_calls"], + }, + } + ], + } + if case.litellm_endpoint == "/v1/messages": + tools: Final = tuple(item for item in items(expected["content"]) if item["type"] == "tool_use") + return {**expected, "content": [*({"type": "text", "text": text} for text in COMMENTARY), *tools]} + assert case.litellm_endpoint == "/v1/responses", case.litellm_endpoint + calls: Final = tuple(item for item in items(expected["output"]) if item["type"] == "function_call") + return {**expected, "output": [*(dict(item) for item in MESSAGE_ITEMS), *calls]} diff --git a/tests/integration/translation/responses/cross_model/test_responses_cross_model.py b/tests/integration/translation/responses/cross_model/test_responses_cross_model.py new file mode 100644 index 00000000000..cb6f973d2df --- /dev/null +++ b/tests/integration/translation/responses/cross_model/test_responses_cross_model.py @@ -0,0 +1,135 @@ +from dataclasses import replace +from pathlib import Path +from typing import Final +from unittest.mock import ANY +from uuid import uuid4 + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.provider import SharedProvider +from integration._support.responses_vendor import same_response +from integration.translation.case import TranslationTestCase +from integration.translation.response_contracts import responses_structure +from integration.translation.response_scenarios import two_messages_before_tools +from integration.translation.responses.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.responses.bases.azure_ai import ( + CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE, +) +from integration.translation.responses.bases.openai_responses import GPT_5_6_SOL_TEST_CASE, GPT_6_1_SOL_TEST_CASE +from integration.translation.runner import assert_translation + +GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE) +AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( + AZURE_CLAUDE_SONNET_5_5_TEST_CASE, + scenario="cross_model_rich_request", + litellm_request={**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, "model": "azure_ai/claude-sonnet-5-5"}, + expected_provider_request=CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, + mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + expected_litellm_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, + "model": "azure_ai/claude-sonnet-5-5", + }, +) +GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE: Final = replace( + GPT_5_6_SOL_TEST_CASE, + scenario="previous_response_id", + litellm_request={**GPT_5_6_SOL_TEST_CASE.litellm_request, "model": "openai/responses/gpt-6.1-sol"}, + expected_provider_request={**GPT_5_6_SOL_TEST_CASE.expected_provider_request, "model": "gpt-6.1-sol"}, + mock_provider_response={**GPT_5_6_SOL_TEST_CASE.mock_provider_response, "model": "gpt-6.1-sol"}, + expected_litellm_response={ + **GPT_5_6_SOL_TEST_CASE.expected_litellm_response, + "model": GPT_6_1_SOL_TEST_CASE.expected_litellm_response["model"], + }, +) + + +@pytest.mark.parametrize( + "case", + (CLAUDE_SONNET_5_5_TEST_CASE, AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE), + ids=(CLAUDE_SONNET_5_5_TEST_CASE.id, AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE.id), +) +def test_responses_keep_the_same_public_contract_on_native_and_bridged_routes( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert {key: value for key, value in case.litellm_request.items() if key != "model"} == { + key: value for key, value in GPT_6_1_SOL_TEST_CASE.litellm_request.items() if key != "model" + } + native: Final = assert_translation(GPT_6_1_SOL_TEST_CASE, gateway, provider) + compared: Final = assert_translation(case, gateway, provider) + assert responses_structure(compared) == responses_structure(native) + + +def test_responses_preserves_multiple_message_items_before_tools(gateway: Gateway, provider: SharedProvider) -> None: + body: Final = assert_translation(GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE, gateway, provider) + responses_structure(body) + + +@pytest.mark.parametrize( + "case", + (GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE, AZURE_CLAUDE_SONNET_5_5_TEST_CASE), + ids=(GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE.id, AZURE_CLAUDE_SONNET_5_5_TEST_CASE.id), +) +def test_previous_response_id_replays_the_turn_without_leaking_internal_routing_fields( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider, tmp_path: Path +) -> None: + stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = tmp_path / "stored-responses.yaml" + config.write_text( + yaml.safe_dump( + { + **stock, + "general_settings": {**object_value(stock["general_settings"]), "store_prompts_in_spend_logs": True}, + } + ) + ) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + assert_previous_response_replay(case, owned.gateway, provider) + + +def assert_previous_response_replay(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + identity: Final = f"resp_contract_{uuid4().hex}" + first_case: Final = replace( + case, + mock_provider_response={**case.mock_provider_response, "id": identity}, + expected_litellm_response={**case.expected_litellm_response, "id": ANY}, + ) + first: Final = assert_translation(first_case, gateway, provider) + responses_structure(first) + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', + (string_value(case.litellm_request["model"]),), + ), + lambda rows: any(same_response(str(row["request_id"]), str(first["id"])) for row in rows), + seconds=70, + ) + previous_identifier: Final = next( + str(row["request_id"]) for row in rows if same_response(str(row["request_id"]), str(first["id"])) + ) + expected_request: Final = ( + {**case.expected_provider_request, "input": "Say hello again.", "previous_response_id": identity} + if case is GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE + else { + **case.expected_provider_request, + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Say hello."}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hello."}]}, + {"role": "user", "content": [{"type": "text", "text": "Say hello again."}]}, + ], + } + ) + followup: Final = replace( + first_case, + litellm_request={**case.litellm_request, "input": "Say hello again.", "previous_response_id": first["id"]}, + expected_provider_request=expected_request, + mock_provider_response={**case.mock_provider_response, "id": f"{identity}_second"}, + expected_litellm_response={ + **case.expected_litellm_response, + "id": ANY, + "previous_response_id": None if case is GPT_6_1_SOL_PREVIOUS_RESPONSE_ID_TEST_CASE else previous_identifier, + }, + ) + responses_structure(assert_translation(followup, gateway, provider)) diff --git a/tests/integration/translation/runner.py b/tests/integration/translation/runner.py index 147409179d6..a5470df7e2d 100644 --- a/tests/integration/translation/runner.py +++ b/tests/integration/translation/runner.py @@ -34,9 +34,15 @@ def expected_response_for_deployment( } -def assert_translation(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> dict[str, JsonValue]: +def assert_translation( + case: TranslationTestCase, + gateway: Gateway, + provider: SharedProvider, + *, + headers: Mapping[str, str] | None = None, +) -> dict[str, JsonValue]: provider.expect(Reply(status=case.mock_provider_status_code, body=json.dumps(case.mock_provider_response).encode())) - response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request) + response: Final = gateway.request("POST", case.litellm_endpoint, case.litellm_request, headers=headers) received: Final = provider.received() assert_provider_request( received,