From 7b7604f0eda4c033b205e949913d53697b414cca Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 14 Oct 2026 19:53:09 -0700 Subject: [PATCH] test(translation): share rich fixtures across Claude, GPT and Gemini providers (#46585) * test(integration): add rich anthropic claude haiku 5.5, opus 5.5 and fable 5.1 translation cases Capture-backed basic cases on /v1/chat/completions, /v1/messages and /v1/responses for claude-haiku-5-5, claude-opus-5-5 and claude-fable-5-1, reusing the shared client templates and ANTHROPIC_MESSAGES_REQUEST/RESPONSE with model-specific id, content and usage overrides. The minimal Opus 5.5 hello cases are replaced by the rich conversation. Sonnet 5.5 and legacy models are unchanged * test(translation): rich OpenAI captures for gpt-6.1-sol, gpt-6-luna, gpt-6-astra and gpt-5.6-terra Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(translation): rich Azure gpt-6.1-sol, gpt-6-luna, gpt-6-astra and gpt-5.6-terra cases * test(translation): rich azure_ai Claude cases for haiku-5-5 and fable-5-1 Bind the shared chat_completions, messages and responses client templates and the anthropic provider templates to azure_ai/claude-haiku-5-5 and azure_ai/claude-fable-5-1 from real Azure AI Foundry captures, keeping the Azure dialect exact (/anthropic/v1/messages, x-api-key plus api-key on chat_completions and responses, system as a text block list on /v1/messages). Add prompt_cache_hit variants for both models from the second call inside the cache window. The two deployments need rows in tests/integration/proxy_config.yaml Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(translation): rich Gemini bases for gemini-3.8-flash, gemini-3.1-pro-preview and gemini-4-argon Add the shared generateContent provider templates in templates/gemini.py from real captures and bind the three target models on chat completions, messages and responses with explicit per-model response overrides. gemini-4-argon is registered in the three basic lists. gemini-3.5-flash is unchanged * test(translation): enrich vertex_ai claude and gemini cases from live captures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): rich Bedrock converse, invoke and mantle translation cases from real captures * test(integration): register the rich translation deployments and follow the rich Gemini 3.8 Flash cases Add the synthetic deployments the rich provider cases bind (anthropic, openai, azure, azure_ai, gemini, vertex_ai, bedrock converse, invoke and mantle) to tests/integration/proxy_config.yaml, all pointed at the fake provider on 127.0.0.1:8191 with synthetic keys. The Gemini omitted-temperature contract now takes its expected generationConfig and reply text from the rich GEMINI_3_8_FLASH_TEST_CASE literals instead of the retired minimal hello cases * test(translation): reuse bridged and native response templates --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry --- .../test_gemini_omitted_temperature.py | 17 +- tests/integration/proxy_config.yaml | 43 +- .../chat_completions/bases/anthropic.py | 465 +++-- .../chat_completions/bases/azure.py | 909 +++++---- .../chat_completions/bases/azure_ai.py | 330 ++-- .../bases/bedrock_converse.py | 636 ++++-- .../chat_completions/bases/bedrock_invoke.py | 555 ++++-- .../chat_completions/bases/bedrock_mantle.py | 678 ++++--- .../chat_completions/bases/gemini.py | 541 ++++-- .../chat_completions/bases/openai.py | 734 +++++-- .../bases/openai_responses.py | 197 +- .../chat_completions/bases/vertex_ai.py | 1375 ++++++++----- .../test_chat_completions_basic_anthropic.py | 2 +- .../test_chat_completions_basic_azure.py | 2 + .../test_chat_completions_basic_azure_ai.py | 2 +- ...chat_completions_basic_bedrock_converse.py | 2 +- ...t_chat_completions_basic_bedrock_invoke.py | 2 +- ...t_chat_completions_basic_bedrock_mantle.py | 4 + .../test_chat_completions_basic_gemini.py | 8 +- .../test_chat_completions_basic_openai.py | 2 + .../test_chat_completions_basic_vertex_ai.py | 11 +- ...hat_completions_prompt_caching_azure_ai.py | 215 ++ .../test_chat_completions_sampling_gemini.py | 225 +++ .../translation/messages/bases/anthropic.py | 780 +++----- .../translation/messages/bases/azure.py | 707 ++++--- .../translation/messages/bases/azure_ai.py | 241 ++- .../messages/bases/bedrock_converse.py | 517 +++-- .../messages/bases/bedrock_invoke.py | 478 +++-- .../messages/bases/bedrock_mantle.py | 475 +++-- .../translation/messages/bases/gemini.py | 526 ++++- .../translation/messages/bases/openai.py | 652 ++++--- .../messages/bases/openai_responses.py | 176 +- .../translation/messages/bases/vertex_ai.py | 990 ++++++---- .../basic/test_messages_basic_anthropic.py | 2 +- .../basic/test_messages_basic_azure.py | 2 + .../basic/test_messages_basic_azure_ai.py | 2 +- .../test_messages_basic_bedrock_converse.py | 2 +- .../test_messages_basic_bedrock_invoke.py | 2 +- .../test_messages_basic_bedrock_mantle.py | 4 + .../basic/test_messages_basic_gemini.py | 8 +- .../basic/test_messages_basic_openai.py | 2 + .../basic/test_messages_basic_vertex_ai.py | 11 +- .../test_messages_prompt_caching_azure_ai.py | 126 ++ .../translation/messages/templates.py | 5 + .../translation/responses/bases/anthropic.py | 550 ++++-- .../translation/responses/bases/azure.py | 1338 ++++++++----- .../translation/responses/bases/azure_ai.py | 386 ++-- .../responses/bases/bedrock_converse.py | 791 +++++--- .../responses/bases/bedrock_invoke.py | 753 ++++--- .../responses/bases/bedrock_mantle.py | 1113 ++++++++--- .../translation/responses/bases/gemini.py | 676 +++++-- .../translation/responses/bases/openai.py | 761 +++++--- .../responses/bases/openai_responses.py | 176 +- .../translation/responses/bases/vertex_ai.py | 1729 +++++++++++------ .../basic/test_responses_basic_anthropic.py | 2 +- .../basic/test_responses_basic_azure.py | 2 + .../basic/test_responses_basic_azure_ai.py | 2 +- .../test_responses_basic_bedrock_converse.py | 2 +- .../test_responses_basic_bedrock_invoke.py | 2 +- .../test_responses_basic_bedrock_mantle.py | 4 + .../basic/test_responses_basic_gemini.py | 8 +- .../basic/test_responses_basic_openai.py | 2 + .../basic/test_responses_basic_vertex_ai.py | 11 +- .../test_responses_prompt_caching_azure_ai.py | 210 ++ .../translation/responses/templates.py | 5 + .../translation/templates/azure.py | 354 ++++ .../translation/templates/bedrock_converse.py | 159 ++ .../translation/templates/bedrock_invoke.py | 144 ++ .../translation/templates/bedrock_mantle.py | 335 ++++ .../translation/templates/gemini.py | 257 +++ .../translation/templates/openai.py | 371 ++++ .../translation/templates/vertex_ai.py | 281 +++ 72 files changed, 15953 insertions(+), 7134 deletions(-) create mode 100644 tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_azure_ai.py create mode 100644 tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_azure_ai.py create mode 100644 tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_azure_ai.py create mode 100644 tests/integration/translation/templates/azure.py create mode 100644 tests/integration/translation/templates/bedrock_converse.py create mode 100644 tests/integration/translation/templates/bedrock_invoke.py create mode 100644 tests/integration/translation/templates/bedrock_mantle.py create mode 100644 tests/integration/translation/templates/gemini.py create mode 100644 tests/integration/translation/templates/openai.py create mode 100644 tests/integration/translation/templates/vertex_ai.py diff --git a/tests/integration/providers/test_gemini_omitted_temperature.py b/tests/integration/providers/test_gemini_omitted_temperature.py index fa8a9c890c6..41488d90f99 100644 --- a/tests/integration/providers/test_gemini_omitted_temperature.py +++ b/tests/integration/providers/test_gemini_omitted_temperature.py @@ -20,9 +20,9 @@ from pydantic import JsonValue GENERATE_TARGET: Final = "/models/gemini-3.8-flash:generateContent" STREAM_TARGET: Final = "/models/gemini-3.8-flash:streamGenerateContent?alt=sse" -EXPECTED_GENERATION_CONFIG: Final = {"max_output_tokens": 1024} ENDPOINT_CASES: Final = (CHAT_COMPLETIONS_TEST_CASE, MESSAGES_TEST_CASE, RESPONSES_TEST_CASE) BURST: Final = ENDPOINT_CASES * 4 +BURST_ANSWER: Final = "I need to inspect `shop/validation.py` next." STREAM_USAGE: Final = { "promptTokenCount": 10, "candidatesTokenCount": 2, @@ -54,6 +54,12 @@ def _generation_config(request: Request) -> dict[str, JsonValue]: return object_value(JSON_OBJECT.validate_json(request.body)["generationConfig"]) +def _expected_generation_config(case: TranslationTestCase) -> dict[str, JsonValue]: + expected: Final = object_value(case.expected_provider_request["generationConfig"]) + assert "temperature" not in expected, expected + return expected + + @pytest.mark.parametrize("case", ENDPOINT_CASES, ids=lambda case: case.litellm_endpoint) def test_streamed_request_without_temperature_forwards_none( case: TranslationTestCase, gateway: Gateway, provider: SharedProvider @@ -64,7 +70,7 @@ def test_streamed_request_without_temperature_forwards_none( assert "Hello." in response.text, response.text received: Final = provider.received() assert [(request.method, request.target) for request in received] == [("POST", STREAM_TARGET)] - assert _generation_config(received[0]) == EXPECTED_GENERATION_CONFIG + assert _generation_config(received[0]) == _expected_generation_config(case) def test_concurrent_mixed_endpoint_burst_without_temperature_forwards_none( @@ -77,7 +83,10 @@ def test_concurrent_mixed_endpoint_burst_without_temperature_forwards_none( pool.map(lambda case: gateway.request("POST", case.litellm_endpoint, case.litellm_request), BURST) ) assert [response.status_code for response in responses] == [200] * len(BURST), [r.text for r in responses] - assert all("Hello." in response.text for response in responses), [r.text for r in responses] + assert all(BURST_ANSWER in response.text for response in responses), [r.text for r in responses] received: Final = provider.received() assert [(request.method, request.target) for request in received] == [("POST", GENERATE_TARGET)] * len(BURST) - assert [_generation_config(request) for request in received] == [EXPECTED_GENERATION_CONFIG] * len(BURST) + expected_configs: Final = [_expected_generation_config(CHAT_COMPLETIONS_TEST_CASE)] * len(BURST) + assert sorted(map(json.dumps, (_generation_config(r) for r in received))) == sorted( + map(json.dumps, expected_configs) + ) diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 91f53f18d2a..171ded4dc52 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -54,6 +54,11 @@ model_list: model: gemini/gemini-3.1-pro-preview api_base: http://127.0.0.1:8191 api_key: synthetic-gemini-key + - model_name: gemini/gemini-4-argon + litellm_params: + model: gemini/gemini-4-argon + api_base: http://127.0.0.1:8191 + api_key: synthetic-gemini-key - model_name: azure_ai/claude-haiku-4-5 litellm_params: model: azure_ai/claude-haiku-4-5 @@ -174,14 +179,19 @@ model_list: model: openai/gpt-6-luna api_base: http://127.0.0.1:8191 api_key: synthetic-openai-key + - model_name: openai/gpt-6.1-sol + litellm_params: + model: openai/gpt-6.1-sol + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key - model_name: openai/gpt-6-astra litellm_params: model: openai/gpt-6-astra api_base: http://127.0.0.1:8191 api_key: synthetic-openai-key - - model_name: openai/gpt-6.1-sol + - model_name: openai/gpt-5.6-terra litellm_params: - model: openai/gpt-6.1-sol + model: openai/gpt-5.6-terra api_base: http://127.0.0.1:8191 api_key: synthetic-openai-key - model_name: openai/responses/gpt-5.6-sol @@ -218,15 +228,21 @@ model_list: api_base: http://127.0.0.1:8191 api_key: synthetic-azure-key api_version: "2025-04-01-preview" + - model_name: azure/gpt-6.1-sol + litellm_params: + model: azure/gpt-6.1-sol + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-key + api_version: "2025-04-01-preview" - model_name: azure/gpt-6-astra litellm_params: model: azure/gpt-6-astra api_base: http://127.0.0.1:8191 api_key: synthetic-azure-key api_version: "2025-04-01-preview" - - model_name: azure/gpt-6.1-sol + - model_name: azure/gpt-5.6-terra litellm_params: - model: azure/gpt-6.1-sol + model: azure/gpt-5.6-terra api_base: http://127.0.0.1:8191 api_key: synthetic-azure-key api_version: "2025-04-01-preview" @@ -348,6 +364,13 @@ model_list: vertex_project: synthetic-vertex-project vertex_location: global vertex_credentials: '{"type": "external_account", "audience": "synthetic-vertex-audience", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "http://127.0.0.1:8190/_oauth/token", "credential_source": {"url": "http://127.0.0.1:8190/health"}}' + - model_name: vertex_ai/gemini-4-argon + litellm_params: + model: vertex_ai/gemini-4-argon + api_base: http://127.0.0.1:8191 + vertex_project: synthetic-vertex-project + vertex_location: global + vertex_credentials: '{"type": "external_account", "audience": "synthetic-vertex-audience", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "http://127.0.0.1:8190/_oauth/token", "credential_source": {"url": "http://127.0.0.1:8190/health"}}' - model_name: bedrock_mantle/anthropic.claude-haiku-4-5 litellm_params: model: bedrock_mantle/anthropic.claude-haiku-4-5 @@ -396,6 +419,18 @@ model_list: api_base: http://127.0.0.1:8191 api_key: synthetic-bedrock-mantle-key aws_region_name: us-east-1 + - model_name: bedrock_mantle/openai.gpt-6-astra + litellm_params: + model: bedrock_mantle/openai.gpt-6-astra + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-mantle-key + aws_region_name: us-east-1 + - model_name: bedrock_mantle/openai.gpt-5.6-terra + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-terra + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-mantle-key + aws_region_name: us-east-1 - model_name: perplexity/pplx-decider-v1-27b litellm_params: model: perplexity/pplx-decider-v1-27b diff --git a/tests/integration/translation/chat_completions/bases/anthropic.py b/tests/integration/translation/chat_completions/bases/anthropic.py index ad6c08126fc..e80dd816477 100644 --- a/tests/integration/translation/chat_completions/bases/anthropic.py +++ b/tests/integration/translation/chat_completions/bases/anthropic.py @@ -86,89 +86,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "anthropic/claude-haiku-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-haiku-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_011CfoboJTmCCdzhHCYYzodZ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "anthropic/claude-haiku-5-5", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, - } - ], - "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "inference_geo": "global", - "service_tier": "standard", - }, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", @@ -429,81 +346,144 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( expected_litellm_response={**CHAT_COMPLETIONS_RESPONSE, "model": "anthropic/claude-sonnet-5-5"}, ) -CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSoggKEAgSGAI4AUIIdGhpbmtpbmcSDHAtt0k5pW+O+4p0yhoMaSOEZS5bpcu05veLIjDPL1g9WsQtCQvCTT+d7CtZXdj8Ky/U+qJXk5YK" + "V+PIuWEZQeNKcaPTleGe9utaR6QqvwdoSvSgT9fVmK4SrR/XSTbPcPpCcRV84Y8g7ZDohrCzpb826nqiRF1A9tQ1K9OrlqiBEmG9ODXI3oy2" + "ag915CwW8oJBjCWAmjmRvkQf64/0IOhqWEunbBPwdfCqqiRtbv1qnqaT2v+WfJeVd7Q72GMGVPjDpI8sZw+k5BoaczYZtJ+T0PbMMNXEwDG8" + "q780JLCmcw5fByE+mX3cUX7SSoAehpwvvmd1/BkzZ7EQaWic7G759KgxzYRtBOiF8Mg9N/holW5GfDF3eBEhAH7JgnV1t+yY+oPy1LpzYGJb" + "GP8gJryTIDRfJmEIX9LwrGjcO30rfGemRv/kzKy1PAY1oLB1jHY6jbO/uGYxZRJy51Sm2Byqj4BHUhZKEEV2GmKwxvr4CBYObD3dBF32FlxD" + "pF4Xz6KuGGUx88ftC51Azp6ySLT/kOsVfqHcXYpH/RbO3BozdCg1AK3pceCljkGmlvZDEVgXZh66O41ZGi7Ku+KbvHADhfJ1dBYodZzNJg8F" + "+MVcQLy4ZhAyfcxp91g6l46vjyEhLEoRNKwrg/dh3cavCsu0EBFQtUgd61U6Eovzyh1JOWDd5mnRE/rN4ISXWcOMKd8o05LZpzmoLkbnub8R" + "DPV0LnPzMd+HHrDpMGoO79qDJU1MdiKhPKGacRP2GnA9Bv5T74r3Y6Zz3KO1tj9TtQ93aPQuiU5Te45T1aIzkNAemkDwuDN/8h/hd0CRCklk" + "2XLb+7ZhQAINBc+yrwoDMq4+vB6y5qjsBBeOMGgWzyPvimWOt4VZczacLJRTKSocioZSYD+tJycCVbaeny41gTGPKicjI4CAji6MEcloNIVt" + "G0bcQwPamtpDokxo/XKQL6W7sL37DQRU5UD+mNG9ntVMgQPb6EmC4boMq/EoEAEqbok0WwUwWhxPfN0EE87gtYUxavNx2/VQJuAVyM8tdKeK" + "hvO8fy0G9d28tRO6yCFvbpKBYDuscZexVT8y5PLJOqVyqIdTYOG6pDDp8MyQ95mDtPHOpTLHo8XAsYtQlHi7qLKmkFx6BjuHu/r97FH2pdgt" + "F2KT5CQmELeTJ1uXD/PsGN60qwONfsWD3fhrrP+rXo32utPzSNM6ndXNfSHHWwKUQ2YyJ3Z/aZCcDFW3e2FkjtcXuhzVL19h3tONpUDyXYOt" + "BGecFGhVULxeQLMjhdK+6gibmoyjgQk/B/F7SO6SJjdvrtgwuKbOEYjSYJUC/49n4CE1gzNdtKdeI2qvYMyADizfeL15vU8mhdoFlOdlPCKd" + "KY+uwXEHfI4/mRgB" +) + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "anthropic/claude-opus-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "anthropic/claude-haiku-5-5"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", "anthropic-version": "2023-06-01", "content-type": "application/json", }, - expected_provider_request={ - "model": "claude-opus-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, + expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-haiku-5-5"}, mock_provider_response={ - "model": "claude-opus-5-5", - "id": "msg_011CfgDzp3URQ38dAAc8F3RP", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-haiku-5-5", + "id": "msg_011Cg36zdxXCpWpnwgbTX1Ss", + "content": [ + { + "type": "thinking", + "thinking": "There's a mention of a chat_completions client I don't actually have access to, " + "so I'll ignore that and stick to my available tools. I'll proceed by reading " + "validation.py, keeping in mind edge cases like zero, negative, and non-integer " + "values before deciding what to examine next.\n" + "\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` to see how `validate_quantity` treats zero, " + "negative, and non-integer values.", + }, + { + "type": "tool_use", + "id": "toolu_015ndrvU4VqqoMBzK35jt4TX", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 288, + "output_tokens_details": {"thinking_tokens": 149}, "service_tier": "standard", "inference_geo": "global", }, - "diagnostics": None, }, expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "anthropic/claude-opus-5-5", - "object": "chat.completion", + **CHAT_COMPLETIONS_RESPONSE, + "model": "anthropic/claude-haiku-5-5", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", + "content": "I'll read `shop/validation.py` to see how `validate_quantity` treats zero, " + "negative, and non-integer values.", "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "tool_calls": [ + { + "index": 2, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_015ndrvU4VqqoMBzK35jt4TX", + "type": "function", + }, + ], + "reasoning_content": "There's a mention of a chat_completions client I don't actually have access " + "to, so I'll ignore that and stick to my available tools. I'll proceed by " + "reading validation.py, keeping in mind edge cases like zero, negative, and " + "non-integer values before deciding what to examine next.\n" + "\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "There's a mention of a chat_completions client I don't actually have " + "access to, so I'll ignore that and stick to my available tools. I'll " + "proceed by reading validation.py, keeping in mind edge cases like " + "zero, negative, and non-integer values before deciding what to " + "examine next.\n" + "\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "There's a mention of a chat_completions client I don't actually " + "have access to, so I'll ignore that and stick to my available " + "tools. I'll proceed by reading validation.py, keeping in mind " + "edge cases like zero, negative, and non-integer values before " + "deciding what to examine next.\n" + "\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + }, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 288, + "prompt_tokens": 880, + "total_tokens": 1168, + "completion_tokens_details": {"reasoning_tokens": 149, "text_tokens": 139}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", @@ -511,82 +491,219 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( }, ) +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSqAYKEAgSGAI4AUIIdGhpbmtpbmcSDBSDqizSHGEEWmpWOxoM4DGvp9vRY4pul7dSIjB07rrpoEV7n5wqYphj7CiaeVIq2s4AlZ2SwEWL" + "xmrdDsZmUBGnz3qCIj9GC21mC5QqxQWhWTlPxM+tJt925MxVve4HmM6e+Gydxpty4kFx7Pk8nqlfDTooB3vyVOyto6ACj/sooxJ1BgEGJgOX" + "z1cH1Y9TG8Y1ca6DiiVm5Gw+6W5/U2FSWZyejNWbykYLxCdMLiLgH3MlCi5SvfIiQR0wgyT/0/1JpM6nrSmPp+zODEqWpbwrCTcwW3g9jSbe" + "Jvzfhr9pi6RZQj37dXkWzBDstK1La/HQoUSn/Zp8EMa8fjQOclN/VzQbAtbcfnhO+jJpQfmfWWC3UVoXFfSw86v+sX4/yBdZ+r7eXLB+dF8B" + "2mEcsWQuCtWa/EBgFR+1Xwyfaqno9/LpFfmiN58IwiAvTwYVkiwhpoJBY6035QDcXQGZURB2eNEMzg4Ay3lY/bp/ElMQPvjKy58ZihDXGysB" + "PPOinfZYNEeCyctJ9VC2957KTV2GlgNVxJ0No9KnZumvvbad4BdU9Bt9z2VsNahvwPVO+sHb2tNLJUsHpznIIoUfrdaEfdPI7r/UoCRt2KRA" + "OO+ejBYkS/D7otJHkjt/1/wOhEFuSNGLZ9yIsSPtI/qFvklkpptEbVQWd8Hs3cwyhZXBoXkFLr7Lku86jFMj8JuErAerc0zgMPLVsUC1s48L" + "mB5lt2xHuYhoqSYbAtRhLBS47t1YMpPf11bUFjRX72wS4jOb0ZMzXHw2G+yQfFhb+xHkUO4OKrDB3+x3GIbjOraTpmeYN6e9MydUuHJiZFzp" + "wTmrNzfYqHN6nNugAjMBJiPDHOmHn4AzeMJEoNu+PU5B73q5Xi22x0AqOF/D0xFIeDQHhDwnqBfvrVIq+Z0JVuoAwVroAx8y9MiIz5YUPlnn" + "N2njDJs7VfN6JIh/J7Ak+mfOgYt92Ofi7HcKAyg4INNwAp3a9lts21O6sBDugOfVr9VFWmASX/TQ/IelHuHHxNtDIgtOFRVLj1QY1iFAyXfT" + "dzYcGAE=" +) + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "anthropic/claude-opus-5-5"}, + expected_provider_endpoint="/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-anthropic-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-opus-5-5"}, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-opus-5-5", + "id": "msg_011Cg371jh5oo3SEfLpT5Eep", + "content": [ + { + "type": "thinking", + "thinking": "I'll focus on reading the validation file first, around line 18, since that's " + "what was specifically requested before looking at the tests.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "Next I'll open `shop/validation.py` around `validate_quantity` (line 18) to see " + "how it handles zero, negative and non-integer quantities.", + }, + { + "type": "tool_use", + "id": "toolu_01Ew2u9E9j2mkT6RbwSw13uR", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 215, + "output_tokens_details": {"thinking_tokens": 66}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "anthropic/claude-opus-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "Next I'll open `shop/validation.py` around `validate_quantity` (line 18) to " + "see how it handles zero, negative and non-integer quantities.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "name": "read_file", + }, + "id": "toolu_01Ew2u9E9j2mkT6RbwSw13uR", + "type": "function", + }, + ], + "reasoning_content": "I'll focus on reading the validation file first, around line 18, since that's " + "what was specifically requested before looking at the tests.\n" + "\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll focus on reading the validation file first, around line 18, " + "since that's what was specifically requested before looking at the " + "tests.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll focus on reading the validation file first, around line 18, " + "since that's what was specifically requested before looking at " + "the tests.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + }, + }, + }, + ], + "usage": { + "completion_tokens": 215, + "prompt_tokens": 880, + "total_tokens": 1095, + "completion_tokens_details": {"reasoning_tokens": 66, "text_tokens": 149}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "service_tier": "standard", + }, + }, +) CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "anthropic/claude-fable-5-1", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "anthropic/claude-fable-5-1"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", "anthropic-version": "2023-06-01", "content-type": "application/json", }, - expected_provider_request={ - "model": "claude-fable-5-1", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, + expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-fable-5-1"}, mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_011Cg33fMNz7YZzvSR7Ycg5X", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_011Cg372bGMBoXaSKUXjtQcr", + "content": [ + { + "type": "text", + "text": "I'll read `shop/validation.py` around `validate_quantity` to see how zero, " + "negative, and non-integer quantities are handled.", + }, + { + "type": "tool_use", + "id": "toolu_01XMSjzRvHCE753Wx4dU2Vxo", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 143, "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", }, - "diagnostics": None, }, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "anthropic/claude-fable-5-1", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello!", + "content": "I'll read `shop/validation.py` around `validate_quantity` to see how zero, " + "negative, and non-integer quantities are handled.", "role": "assistant", + "tool_calls": [ + { + "index": 1, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "name": "read_file", + }, + "id": "toolu_01XMSjzRvHCE753Wx4dU2Vxo", + "type": "function", + }, + ], "provider_specific_fields": {"citations": None, "thinking_blocks": None}, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 143, + "prompt_tokens": 880, + "total_tokens": 1023, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 143}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", diff --git a/tests/integration/translation/chat_completions/bases/azure.py b/tests/integration/translation/chat_completions/bases/azure.py index e7eb937ea39..15a02d6bed1 100644 --- a/tests/integration/translation/chat_completions/bases/azure.py +++ b/tests/integration/translation/chat_completions/bases/azure.py @@ -1,6 +1,9 @@ from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.azure import AZURE_RESPONSES_BRIDGE_REQUEST, AZURE_RESPONSES_RESPONSE +from pydantic import JsonValue GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -459,464 +462,584 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) +GPT_6_LUNA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 489, "check_offset": 0}, + }, +] + +GPT_6_LUNA_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0CO5Co3fPBWyqHQfJ1ms2Y-BIc2sUtT_XTbjdSYTT6QTAaPmp8ylGto8Y3Of8ATTnesNsPFU" + "7kLy9CJ_Lb2Pl5F7ln4lykRgrtPV-7f-B3FU_zaPLktc2eYem0KqseLyq3kBK90cXXTU0eEKJk458iGU" + "fjCFXwxXSB2q_f6rHwxaw5XjQ6IJO_-weFQqarHq3kH7oxqfiWR2udVzYFvkmJ3ezaEQBAJFGREpqHVq" + "NguzyFGuJLNK4urfedbSgOBmOMWZ8jxlPVCpe0FBu4fqvHhKKrBUxBdgvFS9UvhCCAeULfLQrxFBdQk_" + "ncDkur6n1iT9UuFTkSDEX36MdI9i-Wfx7VGbgmc6sZHJMoHEA5Ut11GmiFdQEgYG5R29Nl1iqAjjBm2g" + "3ZN2QNYQ7Ez_FLbOgE4dfF5uw5R239QLURkuEYdPAkfHVfewFiNyHstpoRkQ7qBcSfuvJZTbxPQPHpBd" + "h6tELbKDrZy0upGR-InCK8MGrl9jpSnAtGT5jhS97Je8kay4zEjcFWW7-Km0IchvPLWU__ftBh38tqmn" + "vprkmBsQpcNDhtx5MLZHEixrcbb-48y92c3_voScqLPDT7vM4sXDh3_8Op6ECc-BBh66af2PKef5wBD2" + "eAiD9G2dl5UAsQalER0rf5s_Us0CzDfZIc1P4I6WrGK9odBayY1WYpVU57ng5ZeTfnoSYzQt1iaiiEEB" + "PdVfQHc1Yc44D-Fk1Ipin7dThHshMn4oiYiCQI1ckRcpeLvPCycASykjaPLJyImrGP7CLgKWJPj1wr5R" + "4XWkGwI5MOI83Y-Zdq3eVUQErpJShokpqITgWVJO-zrClUkwsVbely-9CKjlorLDLTYsAbLNIoJp6xxN" + "AF2qt3QObhtKr7mxyq2idnVgdsk4Xj02P6XI-NERyqEdWOnUf7hDG8ZB75thRXDfKPiGkRFyCikPCV7Y" + "4mpeUIE5pytFjYLLOYEq36_GlEcGFosWPyi1Z_V_suyzC0Wy7KGIRMoGdVyyBZrIpCQvVConXD0AFGsY" + "Mvg4pxnwXK8gk_5N7ALIaGj9KM2X1i8ywKepmDw6Y1OVyfGCPhXyfuQd43ul1XIFczTemXw15PC9q2YX" + "d3FuAXQQbKYycaPm7beOLi1gGehWqlcbYx0ZbJV4bk4VrxD94TGoii8T-hbckP2qeJInXCGRsT-vXw3P" + "mGM_IkXTEJvZD_XmfinP7KPyjHyg-Lnx990ODgAOX0UhPCl58lsRlRq0XWldC8soDN-TKQh1E2j3vEGq" + "YC8goY-zCX-5prx9F__fFBYFMQlRPFtDa3JzBEV2cJ8wavsIZ1r9vow6I4QSguIsp5tToakfVaoou4ct" + "p94gqzyuymCtslLZ1Ndau4npTMUT1TzhVbQ-KJPRuuXht03dghpmmSe4sBnmpsLJHigktIL7ozhj3hue" + "Gp6s3t5L8_Cx2vtXKxsOmWsDLM7VxA5Lz_lUGIjgWGyA6LFbssAVkjXLz_okVdFMvDUu2ZkYUnfvwXv1" + "A1sUGJ_259aAEeIu0jlpMhCS6X54" +) + GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "azure/gpt-6-luna", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/deployments/gpt-6-luna/chat/completions?api-version=2025-04-01-preview", - expected_provider_headers={ - "content-type": "application/json", - "api-key": "synthetic-azure-key", - "authorization": "Bearer synthetic-azure-key", - }, - expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "model": "gpt-6-luna", - "max_completion_tokens": 64, - "stream": False, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "azure/gpt-6-luna"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={**AZURE_RESPONSES_BRIDGE_REQUEST, "model": "gpt-6-luna", "stream": False}, mock_provider_response={ - "choices": [ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0257792c0e8dcbb3006ad023b8112081958c7c4379959d4c6b", + "created_at": 1792025528, + "completed_at": 1792025529, + "content_filters": GPT_6_LUNA_CONTENT_FILTERS, + "model": "gpt-6-luna", + "output": [ { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - "finish_reason": "stop", - "index": 0, - "logprobs": None, - "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, - } - ], - "created": 1791231849, - "id": "chatcmpl-EVjWDVYMA8Y26Lw8Jk3617Q3kWSUS", - "model": "gpt-6-luna-2026-09-22", - "object": "chat.completion", - "prompt_filter_results": [ + "id": "rs_0257792c0e8dcbb3006ad023b8e27481959ae045c67273b7d7", + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_LUNA_ENCRYPTED_REASONING, + "summary": [], + }, { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } + "id": "msg_0257792c0e8dcbb3006ad023b939b88195b37db135f24d9db5", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I need `shop/validation.py` next to see the quantity checks, especially how zero " + "and negative values are handled." + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0257792c0e8dcbb3006ad023b97338819589076c2f853d85bb", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":160}', + "call_id": "call_nF1gXonXpNHBt14U2XkvfYZ8", + "name": "read_file", + }, ], - "routing": {"serving_pipereplica": "d20260930053954-f91e8c7b-prefill-r7-dp0-prefill"}, - "service_tier": "default", - "system_fingerprint": None, "usage": { - "completion_tokens": 5, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 2, - "engine_ttft_ms": 65, - "engine_ttlt_ms": 80, - "pre_inference_ms": 83, - "service_tbt_ms": 5, - "service_ttft_ms": 282, - "service_ttlt_ms": 306, - "user_visible_ttft_ms": 199, - }, - "prompt_tokens": 19, - "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, - "total_tokens": 24, + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 129, + "output_tokens_details": {"reasoning_tokens": 65}, + "total_tokens": 441, }, }, expected_litellm_response={ - "id": "chatcmpl-EVjWDVYMA8Y26Lw8Jk3617Q3kWSUS", - "created": 1791231849, + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_0257792c0e8dcbb3006ad023b8112081958c7c4379959d4c6b", "model": "azure/gpt-6-luna", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, - "message": {"content": "Hello!", "role": "assistant", "annotations": [], "refusal": None}, - "provider_specific_fields": { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - } + "message": { + "content": ( + "I need `shop/validation.py` next to see the quantity checks, especially how zero " + "and negative values are handled." + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": ('{"path":"shop/validation.py","start_line":1,"end_line":160}'), + "name": "read_file", + }, + "id": "call_nF1gXonXpNHBt14U2XkvfYZ8", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0257792c0e8dcbb3006ad023b8e27481959ae045c67273b7d7", + "encrypted_content": GPT_6_LUNA_ENCRYPTED_REASONING, + "summary": [], + } + ], }, - "logprobs": None, } ], "usage": { - "completion_tokens": 5, - "prompt_tokens": 19, - "total_tokens": 24, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 2, - "engine_ttft_ms": 65, - "engine_ttlt_ms": 80, - "pre_inference_ms": 83, - "service_tbt_ms": 5, - "service_ttft_ms": 282, - "service_ttlt_ms": 306, - "user_visible_ttft_ms": 199, - }, + "completion_tokens": 129, + "prompt_tokens": 312, + "total_tokens": 441, + "completion_tokens_details": {"reasoning_tokens": 65}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, + "content_filters": GPT_6_LUNA_CONTENT_FILTERS, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, "service_tier": "default", - "prompt_filter_results": [ - { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } - ], - "routing": {"serving_pipereplica": "d20260930053954-f91e8c7b-prefill-r7-dp0-prefill"}, - "system_fingerprint": None, + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], }, ) +GPT_6_1_SOL_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 578, "check_offset": 0}, + }, +] + +GPT_6_1_SOL_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0COvgomuqwo0Z3KFAStLOih2JmYOubPtUjtpD1zCRSCXdDnHGkj8asyVemaCsQr8szo_esDZ" + "PIT2Y9ghPTx1d5moHnS_lyX4nn3Almf4SYZ1pm8maTgEdntGFzcNtPRwW70-CgpeKe5z8zTwxsxyyKDb" + "OYTjt4hjUbAa4pM0K9JR9u5Q_P-LKgfcv3SNCeNSMs5lSV1aBMpuWEGJrlYF020C4wtmru3lH9VjQlOn" + "nW8uD1SCcPm7Sp1_Mm16PJBX4j2grbcIOJZDqj7OcjzRMAb2Gw96QfIg1_efR4BDVHzACkDClzPaiWuI" + "rAmGMA8bJDtQBPijZN3Hahu8eMoMd-XZDVQPx_7XM3F5OWUiuos_kEZNAhIIQHcLrXvyyf7slWvbT4Af" + "c8bvO45t1wOu5E_ALLhWcUCvbfPKBmHrlsRtBEVd_5n0q8jRwR1e5nyjPhMo3YsHvPmwHUJ5Gi1DZycD" + "-FDj6NpwlLcV3eeKaD8c-wNnHuXwfzi_KzIwU1phF_QxJ3ofegtM-3HwCqgaxxVFEhtIh3yezUKeqC7R" + "fSSHpmI-Q59TRf6-R7h6NFJKMG4Hg6C5twvVloHV2YMUHfBrgWr44X9FTzEwxJ89niDRtaB9Xvo2Qzme" + "9LUF-CWuGYTKj5QyRUQFOtclmBL335YTvTDWT8V-eN6KTE33XQNDEPSKuecES6fGHVCuIOeyQkWkvrW8" + "O2OrPJAmoW5g9AFWSWbeLWmDTTav6vbTyu0Ef9cvaRHRTOMq6q8VlEQoTQkJNUM8OKguY7Ad-1VSoIFW" + "D4_LvhuRsDsQiBu1diEq3hRNhXssBUd5wkFLOXw7UITRHONHhk7TDmVh0yjOwBr_e323vmL2e4BouKoX" + "x_0Umeb1jkjFlGgmpW59LmqXd91aXxjioYLxVMOMC7ib7O01_dD-OvWavSxAy9GLnex06mgThS2T5_u4" + "MRmpXIVVqdniqvoxx6YwX5F3TwRoIVc511oXyOG0QkKVLD1k0pI9OdFjiWNVH6XuhWPpVnAhqR6KYejn" + "hQQwuHA4KV4Nd6wOMFbhPjKPCuRcaN1vbgTbRJlwD8O2OV537hjDq6TQaezFJMk3HPK5bB_-J3kmUCul" + "YOXw0jgw-Ze8EmKLDZoCz3jq4NjNA9s0sLGd_Xg0kR6GeOmpLMWtT024yQ7gowGendkcHjIw3S-fIfiI" + "zJtZ-Hk9_iMBk8FE5Syb-j0kmzGF2gFj7ywTTdGoMsM3aTOvusVL14D1N_IfMryUdeoNJ41G8SvqGjtH" + "gyqCER9grGItYgKueDjt6IC5I086__HogkmrpzLZxSIpFCXX6rArVDsIYdS9XFgLW1oBKsVUs5DLnIYe" + "Ncv_cC-oTvEDBaYdhDPYbn_5N6q0q7dtbvohO-XJxOfLh0jC3TelEn7m5EXMPpAygQP2sPtI4_SmhA1C" + "Tji-B-6J60kBUiKmp65ZVxx8BDMI9dv3PLxsw4R0zh6fP_Fsok0UKJOQlfC6k47yhDuTcdN_vq7sAc3q" + "6P9b7Ud7Bz3JtY8Lg516vLK2vti4iwu7dcHlNiIoNZTGMANKYccO-XSRuJbX09bypgJB9sSR56_-Cb1i" + "b1C5RtkTv3ueRvq58Hw1RIINmiTTJn2MiYi_P-yXAirR39YTxcyRNUk4UDgK1lK73lrnpiBELJe-CPQg" + "wgLlalzjk8j0CnIMUA==" +) + GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "azure/gpt-6.1-sol", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/deployments/gpt-6.1-sol/chat/completions?api-version=2025-04-01-preview", - expected_provider_headers={ - "content-type": "application/json", - "api-key": "synthetic-azure-key", - "authorization": "Bearer synthetic-azure-key", - }, - expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "model": "gpt-6.1-sol", - "max_completion_tokens": 64, - "stream": False, - }, - mock_provider_response={ - "choices": [ - { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - "finish_reason": "stop", - "index": 0, - "logprobs": None, - "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, - } - ], - "created": 1791231851, - "id": "chatcmpl-EVjWFBfdrUdRu3rbi6eAXbuO4zfNm", - "model": "gpt-6.1-sol-2026-09-29", - "object": "chat.completion", - "prompt_filter_results": [ - { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } - ], - "routing": {"serving_pipereplica": "d20260929014647-f3a90410-prefill-r11-dp0-prefill"}, - "service_tier": "default", - "system_fingerprint": None, - "usage": { - "completion_tokens": 5, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 8, - "engine_ttft_ms": 440, - "engine_ttlt_ms": 491, - "pre_inference_ms": 70, - "service_tbt_ms": 9, - "service_ttft_ms": 773, - "service_ttlt_ms": 819, - "user_visible_ttft_ms": 703, - }, - "prompt_tokens": 19, - "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, - "total_tokens": 24, - }, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "azure/gpt-6.1-sol"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={**AZURE_RESPONSES_BRIDGE_REQUEST, "model": "gpt-6.1-sol", "stream": False}, + mock_provider_response={**AZURE_RESPONSES_RESPONSE}, expected_litellm_response={ - "id": "chatcmpl-EVjWFBfdrUdRu3rbi6eAXbuO4zfNm", - "created": 1791231851, + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_03952e888f4f1827006ad023aa00108190b24e51d376e9a373", "model": "azure/gpt-6.1-sol", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, - "message": {"content": "Hello!", "role": "assistant", "annotations": [], "refusal": None}, - "provider_specific_fields": { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - } + "message": { + "content": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": ('{"path":"shop/validation.py","start_line":1,"end_line":200}'), + "name": "read_file", + }, + "id": "call_mWfqifVeCXVWLiiNM1Ac2MjL", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_03952e888f4f1827006ad023ae0c048190acdee29b5d20c5b3", + "encrypted_content": GPT_6_1_SOL_ENCRYPTED_REASONING, + "summary": [], + } + ], }, - "logprobs": None, } ], "usage": { - "completion_tokens": 5, - "prompt_tokens": 19, - "total_tokens": 24, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 8, - "engine_ttft_ms": 440, - "engine_ttlt_ms": 491, - "pre_inference_ms": 70, - "service_tbt_ms": 9, - "service_ttft_ms": 773, - "service_ttlt_ms": 819, - "user_visible_ttft_ms": 703, - }, + "completion_tokens": 154, + "prompt_tokens": 312, + "total_tokens": 466, + "completion_tokens_details": {"reasoning_tokens": 89}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, + "content_filters": GPT_6_1_SOL_CONTENT_FILTERS, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, "service_tier": "default", - "prompt_filter_results": [ - { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } - ], - "routing": {"serving_pipereplica": "d20260929014647-f3a90410-prefill-r11-dp0-prefill"}, - "system_fingerprint": None, + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], }, ) +GPT_6_ASTRA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 459, "check_offset": 0}, + } +] + +GPT_6_ASTRA_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0CPDbn4g-dCLkDZ90pK6ePzjtg1TpduhwuSh1HFNOWpCL6JcLlasgA7lAnoba8Zm8LZe5T23" + "BMKHJVWyNRygNF8WDBBQvKvUhX2BOjPSf9wUbkXuctD_VtgwabIhmtwFhS6LH3iYbXzztiGqB898M7sF" + "QPlW8yqde_2hojqasenBXwkhsmeVHZZRxq0TXjc8eZeJ4WEdZOck0TB1eV0xb3UArpyy3amtuONzYCy2" + "wDVDAAlBVPTCcOC67biyQaEL7lipOMukY-yoGoQ-_SnRfLiVQDtl6U2mRA6svav-EEYm8igRyau6CSbp" + "CKYAHflNODMH4y27mX2iCRBeFkZ-nDAzEDdxd7UPFFrp-lKbCueG7JYIyBfkQlJMB_YyY2rcF6AWuLMt" + "K8RauqiMH-lgPZS1p4KhAauH1jSb47s3ZTgF4rowkMLdgAwdxIZD-3aoGXN9l4XOHumUfQNpJMNZbg9c" + "RayPzWlUEOXWEZ7WBchPzl_yIvZCvtTk3ljpUDmJuEb0Ox_TiWqFEr2__UgFccG49x5Sv9pSXqxrsjMw" + "y6ysHzZLbveONQ-gGcraErJcnnwSFSq_gsGiDPV-AJAMo-qY9vf5y6wJOSwIgQHpBx2e3WQAlp2R7vLr" + "QN_-cXtKgTcuVVGGjVLJIazz5Br4Yke70ms5nqPVOQYVfnXTAOzfbOrFfO_U5DoIW6xorPtPd_WFkw7Q" + "jK-gpm7AfkZL2RTICcwZDXuR6RCnSZMbOkBLCFlvZ3WyliM7j3fcvhxhfhA2Svp5kOxVjzZwY0NNMNA9" + "9x5td-Ht8ZgFX-UxIxo8H3sxy5cu-1JKd2eQb4QYoMyVXqk8ZOS28HPGgsx_B4pPTl2KOQ8b0KtNoiB1" + "58YDy9e74nb5u4U9I7Xkf8rS5EGmX1aGFAgW89XEymYCLfBfhtDjZQZAVyPSPxEdQd6CPnoVBy1MGrMK" + "zbyt8X4xEIJJupb8SLuLmfkxKskE4QHfoX5h9As_Ak9WGEn5TIFtJJWmBk6cwZOoswtBSDbgLf-Xid3n" + "2qfPX0UwdUnQzMNdc9eG9CU5vtvs-lRCMJbH3qHlK19vF82EoZT_z4lrHNXSyLOZad1kGaCRDFLakFCW" + "KF7Qil-U69jZCpyaghA1Ebts_es48dSwTLm9Kh323P_y5_o2baqKxT9FUKh5kVztjdiBSLx1LA5mzMUj" + "wdEWQpKh-RfKRnb9YoRGvCYrfvGtF_TcuYEg1DXQkOGejpaeXd-21Vaq44w1XJ72Kyun9KCJvoZu9O01" + "vo8uhUnhNksGqx26p8M6bxIePVhfA-_RVbg_meJov3jgH3OHphRLAO6jIECT-GauSo7p3fjmSviQlkZB" + "520Y5e6n3LPNZ8zpVYvtKooOj1VC0r6F_8vOdN8G5NvjELx2gptVXspfBoM3Ku1q7pFFUKqTG091ZKsS" + "tiFlIphwqI9D4d5aRaerUp2mN_X_aPJoj02O6svBMsbUHAns0PqVoCDg1HogteEQPSiHJTT_D033QDG6" + "ybr4oLs=" +) + GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "azure/gpt-6-astra", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/deployments/gpt-6-astra/chat/completions?api-version=2025-04-01-preview", - expected_provider_headers={ - "content-type": "application/json", - "api-key": "synthetic-azure-key", - "authorization": "Bearer synthetic-azure-key", - }, - expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "model": "gpt-6-astra", - "max_completion_tokens": 64, - "stream": False, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "azure/gpt-6-astra"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={**AZURE_RESPONSES_BRIDGE_REQUEST, "model": "gpt-6-astra", "stream": False}, mock_provider_response={ - "choices": [ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_03d9860d694cf03b006ad023bfd5f881979bc737bdacd3fdec", + "created_at": 1792025535, + "completed_at": 1792025539, + "content_filters": GPT_6_ASTRA_CONTENT_FILTERS, + "model": "gpt-6-astra", + "output": [ { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - "finish_reason": "stop", - "index": 0, - "logprobs": None, - "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, - } - ], - "created": 1792022229, - "id": "chatcmpl-EZ38HKM9wxIe1d1BEJdXW3rW8ceOB", - "model": "gpt-6-astra-2026-09-03", - "object": "chat.completion", - "prompt_filter_results": [ + "id": "rs_03d9860d694cf03b006ad023c113c0819784747d4f0860d98a", + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_ASTRA_ENCRYPTED_REASONING, + "summary": [], + }, { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } + "id": "msg_03d9860d694cf03b006ad023c246808197b42e6d5551340e2b", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I’ll read `shop/validation.py` first to see how it handles zero, negative, and " + "non-integer quantities.\n" + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_03d9860d694cf03b006ad023c2b42081978ef9bf4a11041d28", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_2kSDJH2w71KGgAbbYNfoELU7", + "name": "read_file", + }, ], - "routing": {"serving_pipereplica": "d20260921004409-52a9ff06-prefill-r12-dp0-prefill"}, - "service_tier": "default", - "system_fingerprint": None, "usage": { - "completion_tokens": 5, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 12, - "engine_ttft_ms": 311, - "engine_ttlt_ms": 384, - "pre_inference_ms": 149, - "service_tbt_ms": 6, - "service_ttft_ms": 948, - "service_ttlt_ms": 977, - "user_visible_ttft_ms": 799, - }, - "prompt_tokens": 19, - "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, - "total_tokens": 24, + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 135, + "output_tokens_details": {"reasoning_tokens": 70}, + "total_tokens": 447, }, }, expected_litellm_response={ - "id": "chatcmpl-EZ38HKM9wxIe1d1BEJdXW3rW8ceOB", - "created": 1792022229, + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_03d9860d694cf03b006ad023bfd5f881979bc737bdacd3fdec", "model": "azure/gpt-6-astra", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, - "message": {"content": "Hello!", "role": "assistant", "annotations": [], "refusal": None}, - "provider_specific_fields": { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - } + "message": { + "content": ( + "I’ll read `shop/validation.py` first to see how it handles zero, negative, and " + "non-integer quantities.\n" + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": ('{"path":"shop/validation.py","start_line":1,"end_line":200}'), + "name": "read_file", + }, + "id": "call_2kSDJH2w71KGgAbbYNfoELU7", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_03d9860d694cf03b006ad023c113c0819784747d4f0860d98a", + "encrypted_content": GPT_6_ASTRA_ENCRYPTED_REASONING, + "summary": [], + } + ], }, - "logprobs": None, } ], "usage": { - "completion_tokens": 5, - "prompt_tokens": 19, - "total_tokens": 24, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 12, - "engine_ttft_ms": 311, - "engine_ttlt_ms": 384, - "pre_inference_ms": 149, - "service_tbt_ms": 6, - "service_ttft_ms": 948, - "service_ttlt_ms": 977, - "user_visible_ttft_ms": 799, - }, + "completion_tokens": 135, + "prompt_tokens": 312, + "total_tokens": 447, + "completion_tokens_details": {"reasoning_tokens": 70}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, + "content_filters": GPT_6_ASTRA_CONTENT_FILTERS, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, "service_tier": "default", - "prompt_filter_results": [ + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], + }, +) + +GPT_5_6_TERRA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 388, "check_offset": 0}, + }, +] + +GPT_5_6_TERRA_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0CPOOqy44P30Q1NqlPLLgAVH4x-Gd5CYim_MQz5OgD7eB1qP82U2pKslsZhD5378bFPvj194" + "3FU_ZJKBe0K8qBQ7WC0llLY7GBHxkqG1pk1QoaRjPi5Ns_gwkF3hFSCnGXgjN_6WZVc8mraXVm6kVVCp" + "YVctOcGPC0FEwPkvFk-UKGyVyEn8--LESX2S0O_3ObP1S4beDMv4L8kAF3LOAlGBKujFdfqk1ch1-yZa" + "CTA0ePCYXz35wUGwmjpO2NIo-nMPuDUeP-1Y7jzG8gJBdkP9o7-7pR9aCWcurj8wn-ae3Sgtx9WE7I42" + "eiDBp0CAV0uet_SDAZ5csE0v6Q_amiKBh7TEabBnPMp4CYZxjhVRDAiU3oDciHzq8odZB1Ftwr3_YmEX" + "YKtuet_lg2wE6s1yEX4itWaYkT6sfh9lR-DafSdF1o9-2jUbgP4FwYrl3LBSadUYlP-mDlsAwrm3caif" + "GQFATcpLE229i8Yw9aiMmrgWXpJ-vP-f-gM_8EWPZ_HXXTJIrCqYcnD4LjxmLHXesuTdt_QsdeUxSFqX" + "SzH_Ywl0fNqlH_LUtG1PfO8CEG_TF9QNFsDBY9qOE9NO0LHoTbUsjTCcdglewNQUh_s8W2L_PBMqYpxE" + "v8xWdvKTsnO1D0aMsE5ovo_T8ROJR46NrrcdAsFPFEceL5G_yjGwLUnB7TRFRvkSm8BSjnaCKqbv9TPy" + "Uzs3CjMsGmz3_8mHRMJTP8hO0kp97a8oZMKeokVrdSYk4pGv9Oyy9GW1BaUVvCEgUV7Dmrk-mt9znTCf" + "GlbBkiF6x_LqdQahGjVdSJQk2NpVZQktzKNyQhzHS-ATargehB4TAANyvyRUL4-zyK_QLVCaN2AlIMPx" + "_6frDw-vmb6bNHXmpc5fyU66Z0JaL_kTpHVTDBdxD5NprC8DEBaaxbnb283hP15dRxLHAb6fOprCkbLd" + "oJfXyyRmoCD2kkdIBhrzOxFr7h9Ipfvf-Qy-gjRpZVbfOzkUTfqxqe5vI5uAGk8uxz5EQFgdH_NgNaZ7" + "76ji97ewVoX-RWgnAra0kzzU_ECDOE52lztUcz6yKLwH9Rw2FqxZJucA0Bh6_6lQbsZAlWz9CuYTcCtu" + "6lnJVfmsVMOJRvnykjYGzaX5tpShxmsu1YiCRmOzXTxB2wdATSHr90OWH3_8w-OkVDBBUO9ByDxlepid" + "zcX043O9BJhdkXFWBUrwjzBO_NWyKKLm7yRu6Evgg34EvlFhFc1F2p6qqTsml_yNsZyw6I3NErTkw4KH" + "aYVQH-DCiM7QoqJK6eJxaEk8QBF6Macs8aXVCYC1xXqvApLQELhvuIDLcGDQSQYLeJKdbq6UksNnp-LE" + "1bt7HPvcSL3heudkey4BtoE5OqnyDV2msEzLtkw=" +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "azure/gpt-5.6-terra"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={**AZURE_RESPONSES_BRIDGE_REQUEST, "model": "gpt-5.6-terra", "stream": False}, + mock_provider_response={ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0146fd70db485046006ad023cbf5b48197a496e74bfbe99f66", + "created_at": 1792025548, + "completed_at": 1792025550, + "content_filters": GPT_5_6_TERRA_CONTENT_FILTERS, + "model": "gpt-5.6-terra", + "output": [ { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, + "id": "rs_0146fd70db485046006ad023cd55848197b06f3cbd7f6046e5", + "type": "reasoning", + "content": [], + "encrypted_content": GPT_5_6_TERRA_ENCRYPTED_REASONING, + "summary": [], + }, + { + "id": "msg_0146fd70db485046006ad023cddf1c8197bfc55155a0310a0b", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "Next I need `shop/validation.py` to inspect the validation branches for zero, " + "negative, and non-integer quantities before reviewing the tests." + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0146fd70db485046006ad023ce20d481979de3a49e428ae4e8", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_TZsfYd2RtwusZpfe2wrbwnRf", + "name": "read_file", + }, + ], + "usage": { + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 110, + "output_tokens_details": {"reasoning_tokens": 41}, + "total_tokens": 422, + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_0146fd70db485046006ad023cbf5b48197a496e74bfbe99f66", + "model": "azure/gpt-5.6-terra", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": ( + "Next I need `shop/validation.py` to inspect the validation branches for zero, " + "negative, and non-integer quantities before reviewing the tests." + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": ('{"path":"shop/validation.py","start_line":1,"end_line":200}'), + "name": "read_file", + }, + "id": "call_TZsfYd2RtwusZpfe2wrbwnRf", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0146fd70db485046006ad023cd55848197b06f3cbd7f6046e5", + "encrypted_content": GPT_5_6_TERRA_ENCRYPTED_REASONING, + "summary": [], + } + ], }, } ], - "routing": {"serving_pipereplica": "d20260921004409-52a9ff06-prefill-r12-dp0-prefill"}, - "system_fingerprint": None, + "usage": { + "completion_tokens": 110, + "prompt_tokens": 312, + "total_tokens": 422, + "completion_tokens_details": {"reasoning_tokens": 41}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, + }, + "content_filters": GPT_5_6_TERRA_CONTENT_FILTERS, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "service_tier": "default", + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], }, ) diff --git a/tests/integration/translation/chat_completions/bases/azure_ai.py b/tests/integration/translation/chat_completions/bases/azure_ai.py index 6e37695e4ff..5ca6cd110b7 100644 --- a/tests/integration/translation/chat_completions/bases/azure_ai.py +++ b/tests/integration/translation/chat_completions/bases/azure_ai.py @@ -2,6 +2,8 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -85,90 +87,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "azure_ai/claude-haiku-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/anthropic/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-azure-ai-key", - "api-key": "synthetic-azure-ai-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-haiku-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_011CfoboMutM3uKRT9qW3upD", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "azure_ai/claude-haiku-5-5", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, - } - ], - "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "inference_geo": "global", - "service_tier": "standard", - }, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", @@ -583,19 +501,27 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( }, ) +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSwwgKEAgSGAI4AUIIdGhpbmtpbmcSDKfqwDliolwjkEk4/xoMbtopFWwu6m8MsKPLIjBYY7k/jbVLmSR7KHeAvyUV9jUyWBV6qdv+M9Uq" + "JG7K7iHhxyzCRoLI+DsRvaqtq3gq4AdEnhxf+3y6bp2Gfgm19WyxAVxsc3BDwyrIOL2frop5j8DtwrGL7KwweKZcQ9Zm6R/VtGDH2r88PsxM" + "g2oSgmYTBR0jEyaOxl4EwRllE3RfE8vkQP66TAx4RyCpbIGVB1lBXxdxVnFgzS8JQMaJnQ4jdXUdR8sH57heOUMuE2trGBVaKoDSWli2FtTw" + "kDNHd99zv/b2cSZtIN7W/d/FuxRVzxlvuuFQO27y7pmaczsSldqKmCRtcGOb21RFF9u3I9eznIN8394JyQWIXEdWmnXuQL/7BKai2nD+R7No" + "5gLT9gY5y1stqQZ8FPkbHxKnt9/WciUc5lyGz7uGJbUiEUYAFR0j41l8+3VXWpF4OV1aEQa2eHNATLpYJ8I3Wctlu2mcOm+096aak+BW1XXt" + "vnkMSvBbM3JUElx5k5chZmfMUHFA32KuAj3VZ58lfnOBxGDE7gMNMVvxdAC/h7nRSrO6ICgMYYXy4kFkpVLZz6kH6K32cEAwl1pCPjxYIZVL" + "OwrPXzjsQUqRpjekBFr1lx23oxR2fUId0EsFpPbJSZHjaN/amBsMY/g3iGFiPU3uQW7+Llvhb/OGpNwI3YtWxi3cTZkNdLaItCnl0QsJ6aFS" + "D0oq0HcrKsIvcbMp05kz/rNFAGRBd5QrUthuh/CvYJUHosauz5OYrvRlzzKlsEVomzIkwVQw0ZrYhq0xZvwIRa7JbRoDadHL6uTNybKFCISX" + "33IimEMVpkaMBEvygWya25H7pSK9OREbzV17rpS5cGBebOLKYjgZms7WhAERsyNm+gYIDcr+X5AGrOrz3qM/diknhnC5rNgbaO8w+JQl1i+D" + "wrNXsCW61u+/KRI2Qf3ekaHG4fcETWz+Y1+JisK7WskmyihKAkexlLaxqyn5RisYGj9tKs4OpjCd1w4lhS6346ypK0ufsfyHjqfn7TVjIsVt" + "posdTv+Wc3fTr7WDcmiJ+hBD/YxouCYYxS+hssBS/ut0QFfWH2dh0/2qGeNd70QqXEt+tbsopsrqpV2OZy9YE8VvLqZicNJHqbZTivXKsEFX" + "YN4pnKv4X7ykWtCILbpbTZNg1iBa4z+T2fSFdAUJVgRFherqjCcD209AA4wj3UJewZX8nwX2Js1W/MRFrAIWzwwBI3x6eCq6OaaeXzSwUS9M" + "I24Sk4o5rm9/czgj2Ottsq0Rmvsng/r3ZOpu8uKfjhxKmCSSQEE4ZZ7NBwLTOUVOcmBoeRfQbyjDKr7NYiaVuN7NlJw2NQbsyhAgqlD8IBpz" + "Z0XczPP0u+uEKTIsVXcjAtHIjyx361hlCa1waGh5DqmWg7zhv2QenFwjBBgB" +) -CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "azure_ai/claude-fable-5-1", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "azure_ai/claude-haiku-5-5"}, expected_provider_endpoint="/anthropic/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-azure-ai-key", @@ -603,63 +529,209 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "anthropic-version": "2023-06-01", "content-type": "application/json", }, - expected_provider_request={ - "model": "claude-fable-5-1", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, + expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-haiku-5-5"}, mock_provider_response={ - "model": "claude-fable-5-1", - "id": "msg_011Cg33iornZJuXHAdrimopu", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-haiku-5-5", + "id": "msg_011Cg37MxTLdwH6PHKZXCEey", + "content": [ + { + "type": "thinking", + "thinking": "I need to read shop/validation.py next, keeping in mind edge cases like zero, " + "negative, or non-integer quantities since the validation logic likely handles those. " + "I'll look at the function starting around line 18 to understand its scope.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` first. The search hit `validate_quantity` at line 18, " + "so I'll start there and check how it treats zero, negative, and non-integer values.", + }, + { + "type": "tool_use", + "id": "toolu_01M68cqREYA4WqXBUK8rqcEq", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 324, + "output_tokens_details": {"thinking_tokens": 167}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "azure_ai/claude-haiku-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll read `shop/validation.py` first. The search hit `validate_quantity` at line 18, " + "so I'll start there and check how it treats zero, negative, and non-integer values.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_01M68cqREYA4WqXBUK8rqcEq", + "type": "function", + }, + ], + "reasoning_content": "I need to read shop/validation.py next, keeping in mind edge cases like zero, " + "negative, or non-integer quantities since the validation logic likely handles those. " + "I'll look at the function starting around line 18 to understand its scope.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I need to read shop/validation.py next, keeping in mind edge cases like zero, " + "negative, or non-integer quantities since the validation logic likely handles those. " + "I'll look at the function starting around line 18 to understand its scope.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I need to read shop/validation.py next, keeping in mind edge cases like zero, " + "negative, or non-integer quantities since the validation logic likely handles those. " + "I'll look at the function starting around line 18 to understand its scope.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + }, + }, + }, + ], + "usage": { + "completion_tokens": 324, + "prompt_tokens": 880, + "total_tokens": 1204, + "completion_tokens_details": {"reasoning_tokens": 167, "text_tokens": 157}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "service_tier": "standard", + }, + }, +) + +CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "azure_ai/claude-fable-5-1"}, + expected_provider_endpoint="/anthropic/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-azure-ai-key", + "api-key": "synthetic-azure-ai-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-fable-5-1"}, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-fable-5-1", + "id": "msg_011Cg37NcfzdXibtC3dWhVWi", + "content": [ + { + "type": "text", + "text": "I'll read the validation code and the test together since they are independent.", + }, + { + "type": "tool_use", + "id": "toolu_01LxkeKf9WX9BVpiWrvfKUyc", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_0139NDXUipoVov5pyoTErFZZ", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 216, "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "not_available", }, - "diagnostics": None, }, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "azure_ai/claude-fable-5-1", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello!", + "content": "I'll read the validation code and the test together since they are independent.", "role": "assistant", + "tool_calls": [ + { + "index": 1, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "name": "read_file", + }, + "id": "toolu_01LxkeKf9WX9BVpiWrvfKUyc", + "type": "function", + }, + { + "index": 2, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}', + "name": "read_file", + }, + "id": "toolu_0139NDXUipoVov5pyoTErFZZ", + "type": "function", + }, + ], "provider_specific_fields": {"citations": None, "thinking_blocks": None}, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 216, + "prompt_tokens": 880, + "total_tokens": 1096, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 216}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, "inference_geo": "not_available", "service_tier": "standard", diff --git a/tests/integration/translation/chat_completions/bases/bedrock_converse.py b/tests/integration/translation/chat_completions/bases/bedrock_converse.py index 32b40ba336d..c46528468de 100644 --- a/tests/integration/translation/chat_completions/bases/bedrock_converse.py +++ b/tests/integration/translation/chat_completions/bases/bedrock_converse.py @@ -2,6 +2,12 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.bedrock_converse import ( + BEDROCK_CONVERSE_REQUEST, + BEDROCK_CONVERSE_RESPONSE, + CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, +) CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -60,64 +66,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/converse", - expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], - }, - mock_provider_response={ - "metrics": {"latencyMs": 954}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", - "usage": { - "cacheReadInputTokenCount": 0, - "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, - "serverToolUsage": {}, - "totalTokens": 29, - }, - }, - expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", - "object": "chat.completion", - "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], - "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - }, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", @@ -289,173 +237,525 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/converse/us.anthropic.claude-haiku-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], - }, - mock_provider_response={ - "metrics": {"latencyMs": 892}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", - "usage": { - "cacheReadInputTokenCount": 0, - "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, - "serverToolUsage": {}, - "totalTokens": 29, - }, - }, + expected_provider_request={**BEDROCK_CONVERSE_REQUEST}, + mock_provider_response={**BEDROCK_CONVERSE_RESPONSE}, expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", - "object": "chat.completion", - "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + **CHAT_COMPLETIONS_RESPONSE, + "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll read `shop/validation.py` next, starting with the `validate_quantity` function " + "found at line 18.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "tooluse_qWzJMc6h58NlspiEa01FUC", + "type": "function", + }, + ], + "reasoning_content": "I need to figure out which file to read for validation, then use a read-only tool to " + "look at it—I'll check the first 60 lines or so since I don't know its exact " + "length.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I need to figure out which file to read for validation, then use a read-only " + "tool to look at it—I'll check the first 60 lines or so since I don't know " + "its exact length.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "reasoningContentBlocks": [ + { + "reasoningText": { + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + "text": "I need to figure out which file to read for validation, then use a " + "read-only tool to look at it—I'll check the first 60 lines or so " + "since I don't know its exact length.\n\n", + }, + }, + ], + }, + }, + }, + ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 230, + "prompt_tokens": 860, + "total_tokens": 1090, + "completion_tokens_details": {"reasoning_tokens": 96, "text_tokens": 134}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 856, + "cache_creation_tokens": 856, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 856, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 856, "cache_read_input_tokens": 0, }, }, ) +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQS9QUKEggSEAEYAjgBQgh0aGlua2luZxIMHeQumEXUSngowYp2Ggw6RETkY6n11eC2W98iMLxQa5+C/eZvHir2PVIhfArk/wZFsCSOH7e3" + "dqBUFaBA8tvcE0LO+g3l1B51aj/2bCqQBbsVVKGduphL2uGxAmAfucVEigmAzobLXjnxMizAS4+QFSE4iS/vbgj9oH/JgaZu1izkc77Y8vtC" + "IzTWqlSeyu8ThzVgljSrPMO76qbBKJPM3N/3VYKrV2iK1OljKXeJMZXuEFTtUJm562vm1sihud8t1EnG+Q849jjXts3Y4sB5tfDewbFG5LsY" + "tclTSq468Kv0fw7vTEGQ1silrl1hMvMc65ZD1o7YAwYurdDaTc1JJbEYLVNtHgwFK4G1G4vdWIHuk9DKNvCeOxFoGhfyYwGbDMX9SqYaHQh3" + "rcX5I/b5JHf2bE2l/5pyiBirAXjlG9EhBOYCdypDGXzlDqr4Oc4xaRWYjv8ixXhov9FHEUj0sdB3YTMxnEaUn4KgvdBSTS+O89wOMcoZBz8T" + "1mR3ig3IxUV27YS6ZSYImaoie9VzuWLNovu1x3rFyrM2cP222Z9NahtcOIXysUVSoe1nYYAYa4K5A7+BXf2xWhUZjfyzG/mGP/LsQCZeXxwY" + "1e0m3ivyUanPUxVrnK1nvpB9FcY7T8UPw2LH2W8VWDXG1MPdjPwZcQJsdc/LxwSmWUv4jwa/jEBOvyKC9GTJRn2TDiUNhzFoLn/wSzdfk5aV" + "rwJd0z5ALiaR9yXm5qgGmK0v1slPo8VmgzoFzLGmkU4UE0uT3etr9IfiR9wAyctSMuKuZ+ApETINbZWOybLaJynLYdaoGYrCwTDB2p3JTQV5" + "HP8aQm6l11YgxxymNnSMHEXmYibBp2B1YH/2TkysI01LMnxBGo79aa1KnzYHSZQpbmqSDmjn6SR3KhjwYxDz7drr8S49a5QPxsYoaffWTrwj" + "0BbxGgQuwjB0pbIQFWiK+eOA+lShN3Sm+8WrdtcwPUbmGAE=" +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={**BEDROCK_CONVERSE_REQUEST}, + mock_provider_response={ + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 86}}}, + "metrics": {"latencyMs": 3449}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + "text": "The user asked me to read the validation file first, so I'll do that " + "before looking at the tests, even though reading both in parallel would be " + "fine too. I'll pull in the full validation file since it's short, roughly " + "18 lines.\n\n", + }, + }, + }, + { + "text": "I'll read `shop/validation.py` first. `validate_quantity` is defined at line 18, " + "so I'm reading the top of the file through the function and what follows it.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "name": "read_file", + "toolUseId": "tooluse_rN6y4cQORe8ESxDQcIl9SR", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, + "usage": { + "cacheDetails": [{"inputTokens": 856, "ttl": "5m"}], + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 856, + "cacheWriteInputTokens": 856, + "inputTokens": 4, + "outputTokens": 237, + "serverToolUsage": {}, + "totalTokens": 1097, + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll read `shop/validation.py` first. `validate_quantity` is defined at line 18, so " + "I'm reading the top of the file through the function and what follows it.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "tooluse_rN6y4cQORe8ESxDQcIl9SR", + "type": "function", + }, + ], + "reasoning_content": "The user asked me to read the validation file first, so I'll do that before looking at " + "the tests, even though reading both in parallel would be fine too. I'll pull in the " + "full validation file since it's short, roughly 18 lines.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user asked me to read the validation file first, so I'll do that before " + "looking at the tests, even though reading both in parallel would be fine too. " + "I'll pull in the full validation file since it's short, roughly 18 lines.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "reasoningContentBlocks": [ + { + "reasoningText": { + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + "text": "The user asked me to read the validation file first, so I'll do that " + "before looking at the tests, even though reading both in parallel " + "would be fine too. I'll pull in the full validation file since it's " + "short, roughly 18 lines.\n\n", + }, + }, + ], + }, + }, + }, + ], + "usage": { + "completion_tokens": 237, + "prompt_tokens": 860, + "total_tokens": 1097, + "completion_tokens_details": {"reasoning_tokens": 86, "text_tokens": 151}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 856, + "cache_creation_tokens": 856, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 856, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 856, + "cache_read_input_tokens": 0, + }, + }, +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSwwUKEggSEAEYAjgBQgh0aGlua2luZxIMeqMdLtmn33y9cIexGgym8fsZQt5P/lSqgpQiMAC92xJoxWLNXHHgGhS+0mdadsD4aDidh2B3" + "XDq8VlyD6g3Heynf1faRQ72efXv4zCreBGbwmU6DTP7IlfHZXPnTXAo979PeGnhoChdOtJ6GKSs/w9rjsAee+Eys02cjJerLDxhf44lHr2iW" + "bxhJQNft2DnosDk3OCMtWx77ksDERl4mcnJplzLJSMTCPO9HSbTkJzWJ5Jj7x0DjznblKBes8q1NPFVV7pqXMB7yo2mPlledoEzt+rYXXiFa" + "NBpuSQ8wvflyON6rwl0UoToTGpUb8rjvjqVkInhX82Y3hIbYgixYzhA9OCh+xMuDQzldRLdM5NgT+9dxHrklL1ZaHuWrpBP12q7qo0j3A7dT" + "/2/JAvGiJqfCcGptfc2RPDK/NgnTxBAbqvUW0VUm8zhg7SDHMC+hunwy4phUc8yBpaJFI5vmthSU++vvyVwNYChs6i0DVdMe6c7Lxzzk3W0Z" + "iAMrgsalHyjK6IzsnwMusiMW64foeMN0OKx3lYSZT9oqp6nDVecBwgH4npD2Pq4PPe7C5S5KVCzvq4guVTCcQM7XUXdck6QzgZrDm/kFp3lz" + "splN7GdOh+PUYAvGvmhnBvPkWKFRPSEcX6vPlbNfhC+ADY8C/oVjjVFjvyBv/aF4BDlY8s4ZV76kmiiyfXQ8P+BpO2QtId1GvlmOq8SmwcAe" + "cMgtz4o0nWWNAaTV8CH+YjswCSe6lscs5bTZzf8724OKBx0lOs+Axn0Lwo5gHekzCimCUpA8leG0dIg6t6FOVRmjcmIoTlh39Xu2LmjrTjNG" + "A8BBS3u+O+R2158XIfII393jsgpVwg8Dn5WIAK8U1LzQDAj7IrhSuN1q2UJG8ZEsi6JmFmWdGXdCi1FVWNkyDBgB" +) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-opus-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/converse/us.anthropic.claude-opus-5-5"}, expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], - }, + expected_provider_request={**BEDROCK_CONVERSE_REQUEST}, mock_provider_response={ - "metrics": {"latencyMs": 10460}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 65}}}, + "metrics": {"latencyMs": 3552}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + "text": "Since the user asked to look at the validation file first, I'll focus on " + "that around line 18 before touching the tests, which can wait since " + "they're independent anyway.\n\n", + }, + }, + }, + { + "text": "Next I'll open `shop/validation.py` around `validate_quantity` (line 18). That " + "should show how zero, negative and non-integer values are handled. After that I'll " + "read the tests.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + "name": "read_file", + "toolUseId": "tooluse_zGYCbMhgYTTAQl5dBHNkKs", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 856, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 856, + "cacheWriteInputTokens": 856, + "inputTokens": 4, + "outputTokens": 223, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1083, }, }, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "bedrock/converse/us.anthropic.claude-opus-5-5", - "object": "chat.completion", - "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "Next I'll open `shop/validation.py` around `validate_quantity` (line 18). That should " + "show how zero, negative and non-integer values are handled. After that I'll read the " + "tests.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "name": "read_file", + }, + "id": "tooluse_zGYCbMhgYTTAQl5dBHNkKs", + "type": "function", + }, + ], + "reasoning_content": "Since the user asked to look at the validation file first, I'll focus on that around " + "line 18 before touching the tests, which can wait since they're independent anyway.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "Since the user asked to look at the validation file first, I'll focus on that " + "around line 18 before touching the tests, which can wait since they're " + "independent anyway.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "reasoningContentBlocks": [ + { + "reasoningText": { + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + "text": "Since the user asked to look at the validation file first, I'll focus " + "on that around line 18 before touching the tests, which can wait since " + "they're independent anyway.\n\n", + }, + }, + ], + }, + }, + }, + ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 223, + "prompt_tokens": 860, + "total_tokens": 1083, + "completion_tokens_details": {"reasoning_tokens": 65, "text_tokens": 158}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 856, + "cache_creation_tokens": 856, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 856, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 856, "cache_read_input_tokens": 0, }, }, ) +CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0: Final = ( + "CAQSrwUKEwgSEAEYAjgBQgluYXJyYXRpb24SDNCGJ8kKyGwJSh36gxoM2/14CeUIPZHnkx/iIjAYx/Gy8bDJhXdDJVKLzNi0l/VKFEq/9uTz" + "mI1Vx1GUXRI6Yj7imDIn3cnUzf5dj6AqyQQhWjFSAqAjMK8gOQ2PqoyPonbTrxwsld+dZM44c+ffNp2YgBSEQyNUlXF+nCYX6kAGwmzSZkKq" + "W8CGWrIH2dB+TzWfz3fsx/GglprifZ0YFvyoMq+Ri6LQnS/GFuqtaJrYhwngV34u1M1jFJv3qw+HvTGmiums0KACTSP8F+vI3OfedHfhaiyz" + "pQnbFie2gqVJOvRNst521omILp8ds5QrBoq2VYnwt7TZEO1IfNfan8OAcetSMUvAYCrW2I3IiZ7lTpMTV2K2Dzyw5NLY7LNrwA/bJuKNbyWI" + "JvFi8zrgIxLAsLNXV0gCLib7qdywHg8k/otUlOSOfHzLaDuwO0BR8OpSjJcm38lUpCg+6twP49FC943vhOJ3t9qR3PSeCEkLdzFEZGUak1Xr" + "sqwK/dfU8R22lGelQK7xwVNGKB6JOQ5pbIA8gUviT8DDQ9nQAaj+HKSYiN3ss3+qD3qPMuBK/XYojp5rbIrpqfoyWogvX9hwWCBFAYC9OUHR" + "199q8fAFQxSRVUKSjCPopCQC9AKouLmeeotr0AI6KKB0deg1MudLkgy6KcKvEgptDqOqJJMpyO+3WxJ5serIbwNgIAbs3Kl9rLsJrmZ3qzfW" + "qw0/yxJGjwLFRLAy2o6Q+6y5kAQA0CsURIjbdvIF7hpHkvlAYdzfX+ApwQY115nAUTkYyliu8myjDBCRXb2mEnAKk7+rQ/tr67/szVV/8B8S" + "3CmxaoFlhHHbwCAHpGZNWPFAOa8hvOPxsewJ+Hr5z34kC4g2QdDdiyPRANAYAQ==" +) CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-fable-5-1", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/converse/us.anthropic.claude-fable-5-1"}, expected_provider_endpoint="/model/us.anthropic.claude-fable-5-1/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], + **BEDROCK_CONVERSE_REQUEST, + "toolConfig": { + "tools": [ + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + "name": "search_files", + "description": "Search source and test files for a literal term.", + }, + }, + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + "name": "read_file", + "description": "Read a range of lines from a project file.", + }, + }, + ], + }, }, mock_provider_response={ - "metrics": {"latencyMs": 4060}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}}, + "metrics": {"latencyMs": 5476}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + "text": "Checking `shop/validation.py` to see how `validate_quantity` handles zero, " + "negative, and non-integer inputs, and pulling in the test file at the same " + "time.\n\n", + }, + }, + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "name": "read_file", + "toolUseId": "tooluse_u1AisurVMo0RZh6mAjHMl2", + "type": "tool_use", + }, + }, + { + "toolUse": { + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + "name": "read_file", + "toolUseId": "tooluse_M4eCGwAM39nEOXlNrXyTx5", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 876, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 876, + "cacheWriteInputTokens": 876, + "inputTokens": 4, + "outputTokens": 267, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1147, }, }, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "bedrock/converse/us.anthropic.claude-fable-5-1", - "object": "chat.completion", - "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello.", "role": "assistant"}}], + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "", + "role": "assistant", + "tool_calls": [ + { + "index": 1, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "tooluse_u1AisurVMo0RZh6mAjHMl2", + "type": "function", + }, + { + "index": 2, + "function": { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}', + "name": "read_file", + }, + "id": "tooluse_M4eCGwAM39nEOXlNrXyTx5", + "type": "function", + }, + ], + "reasoning_content": "Checking `shop/validation.py` to see how `validate_quantity` handles zero, negative, " + "and non-integer inputs, and pulling in the test file at the same time.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "Checking `shop/validation.py` to see how `validate_quantity` handles zero, " + "negative, and non-integer inputs, and pulling in the test file at the same " + "time.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "reasoningContentBlocks": [ + { + "reasoningText": { + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + "text": "Checking `shop/validation.py` to see how `validate_quantity` handles " + "zero, negative, and non-integer inputs, and pulling in the test file " + "at the same time.\n\n", + }, + }, + ], + }, + }, + }, + ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 267, + "prompt_tokens": 880, + "total_tokens": 1147, + "completion_tokens_details": {}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, }, }, diff --git a/tests/integration/translation/chat_completions/bases/bedrock_invoke.py b/tests/integration/translation/chat_completions/bases/bedrock_invoke.py index 4215836e57d..5ce2096f5b6 100644 --- a/tests/integration/translation/chat_completions/bases/bedrock_invoke.py +++ b/tests/integration/translation/chat_completions/bases/bedrock_invoke.py @@ -2,6 +2,12 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.bedrock_invoke import ( + BEDROCK_INVOKE_REQUEST, + BEDROCK_INVOKE_RESPONSE, + CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, +) CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -77,82 +83,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/invoke", - expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_bdrk_prp4tfjg2ibow6km7wkqw5euejxxssl2ngm6wbgjxyg7xd32tkja", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - }, - }, - expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, - } - ], - "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "service_tier": "standard", - }, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", @@ -377,226 +307,435 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/invoke", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={**BEDROCK_INVOKE_REQUEST}, + mock_provider_response={**BEDROCK_INVOKE_RESPONSE}, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll read `shop/validation.py` around the `validate_quantity` definition at line 18. " + "I'm checking how it treats zero, negative, and non-integer values.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_bdrk_01Hfy4YfUU7hZUxEmpp4oDYV", + "type": "function", + }, + ], + "reasoning_content": "I should look at shop/validation.py, focusing on how it handles zero, negative, and " + "non-integer inputs before deciding what to read next. I'll check the first 60 lines or " + "so since I don't know the file's full length yet.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I should look at shop/validation.py, focusing on how it handles zero, " + "negative, and non-integer inputs before deciding what to read next. I'll check " + "the first 60 lines or so since I don't know the file's full length yet.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I should look at shop/validation.py, focusing on how it handles zero, " + "negative, and non-integer inputs before deciding what to read next. I'll " + "check the first 60 lines or so since I don't know the file's full length " + "yet.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + }, + }, + }, + ], + "usage": { + "completion_tokens": 295, + "prompt_tokens": 880, + "total_tokens": 1175, + "completion_tokens_details": {"reasoning_tokens": 143, "text_tokens": 152}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "service_tier": "standard", + }, + }, +) + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSwgUKEggSEAEYAjgBQgh0aGlua2luZxIMwv5o3sIXxVdkUO0MGgxObtbvqBQXfANsnrwiMMS0BVrFyZB1bCUZfx4cUYR/+/9/NfiPwTEU" + "jsGSOJFjxW95CdHDmohqchGNWlIBUirdBEZuOEcmqJBmrwvvX5mho5M6DJskEztgdUjCFIzTM5CXU24my+0yxBnIrFUrgzqaihEmKWmwxSEN" + "rGOtF41225PHpYTpfWHKxn6tay7STRpm8K6x6AXYtYJSWMAa0pvCBrsc8U4jvbuAHhLUqxliQG0WwkbX4UK2/6uGMSwYDMHsh+kvDLa17WhU" + "vjZsKg0ocntDgcMBG4WVxhaCMurK2lhDP14G5q3CdViTQd8TiM/ooU2cx7NrlqHWtB0tr4QuZaXZlTZjLz+wpIPzvLSQvyybecc/LrD+4EGV" + "CbVHTucTj8qEBWNoT7JPnwHIH3nWyspyauyD37m3nE2GcTtyIaNFnyXwFMgPZb8CNphtgc0QIYu6/egzk0msGdhyxWo48D/PMs5uL9FjFIrK" + "xiFgG+nE3ck4ipexy/W69W/G2fzL756UZc3dUpYMj21MbAgMyxqiIemKKaw8w8UTaEDJUw7IxXFXklqjOehMaGJT+YDR6G88bPgfG5FTjegK" + "UBVhR472cBMPIauHwaGz5I6ZuetVBuaNuS1DhFnMkRvj+Z/GdtJANK4EySK28w/n9TTqrkFqwO0LBV9lGUgoubOOUfYM/AE55WD6OEX7HbKi" + "t22c/5TTmH8UuSXG5YbMZ3LP5rnB52zvi/wi53E0jVEQraX4Yf2ueoOrunlngmR/xW9Sq7+CK/9mcO6RuMn9yD8LJxf7/bg+yNij8kFGTOvu" + "B06RdfOrbD2MNAWkvtpdfvKcSzVElSQkltI7zoqnKJ23x8CF66vwj3ThszmGexzvdt4CZtrjw4ZcP6YwGUnOGAE=" +) + CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5"}, expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - }, + expected_provider_request={**BEDROCK_INVOKE_REQUEST}, mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, "model": "claude-sonnet-5-5", - "id": "msg_bdrk_tozg7mwrkjb7fisuua7q5ylekeypufs6he64a2tlncr4bvdg4dha", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_a7sjc6m6nfuwupssre5yi22kzpdwizomnscfdrysperwjop7fe5a", + "content": [ + { + "type": "thinking", + "thinking": "Since the user explicitly asked for the validation file first, I'll prioritize reading " + "that before the tests, focusing on the area around line 18.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I need `shop/validation.py` next. It defines `validate_quantity`, which should show how " + "zero, negative and non-integer values are handled. I'll read the area around line 18.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01LNktDaewoxFywoyzT1eMrE", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 224, + "output_tokens_details": {"thinking_tokens": 69}, "service_tier": "standard", }, }, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", + "content": "I need `shop/validation.py` next. It defines `validate_quantity`, which should show " + "how zero, negative and non-integer values are handled. I'll read the area around line " + "18.", "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_bdrk_01LNktDaewoxFywoyzT1eMrE", + "type": "function", + }, + ], + "reasoning_content": "Since the user explicitly asked for the validation file first, I'll prioritize reading " + "that before the tests, focusing on the area around line 18.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "Since the user explicitly asked for the validation file first, I'll prioritize " + "reading that before the tests, focusing on the area around line 18.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "Since the user explicitly asked for the validation file first, I'll " + "prioritize reading that before the tests, focusing on the area around line " + "18.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + ], + }, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 224, + "prompt_tokens": 880, + "total_tokens": 1104, + "completion_tokens_details": {"reasoning_tokens": 69, "text_tokens": 155}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, "service_tier": "standard", }, }, ) +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSigQKEggSEAEYAjgBQgh0aGlua2luZxIMq3frSfibHHldP8/aGgxVeuAxm7EIGP7vV10iMCQSYTMAj3EpHNaNy9ic8GJApnD9KmWXPGJ6" + "BqKYlRYlNcvpHFdOGC3d8HHYlYWetCqlA/lyfcAC8HgdVosWlvsFkLvgcAmmV8VUSSJCD7VHV1sC8YjChiQ0bO/DWbfCeK3FUNbDwhAJEstb" + "q3Pj4THSbsDtJHlIcukGM/xOqXvfjcisMMPjpF8xuK5ai1cnFehLGWHpdj99zC1nK1E/k4OqHYOhzWfHLMQdbeTgR5FP9cVexDyKI/5mlZw9" + "YxIbOxEjSYAwsR5Q6cIDWjl2ao1M4OhorS/UR1FYp99jwQE5a/zRGOPsFpfPLXR4KCqDKCezAH31LLUkc9iHHgblh5N118kZtGCPRZA8EMmj" + "HsKZEJ8NdZZfok3Jlrlx0F2773aBBdOsX9MyzopsW4Ap/izR7swjR3hfZy6j/7CnvyghiT7N/4rpIDXiz/2XjVdumk3Rddta3PijbpUpDsMJ" + "63IJrMcFYLbDypGIdbdHjNyriHlOiwDUy3t4Yh/CwOgw9gkPLKoAVHStFt7YiNnoNCzCGCcbCav7zTCCBZDLdhvbV3QNDHc+b6KkSLsvN+md" + "u1+4Y6IkqLrhjV7TMledvs22t7vrI5w9M39WGwLztkxmCZbPzfyCWEMYAQ==" +) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-opus-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-opus-5-5"}, expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - }, + expected_provider_request={**BEDROCK_INVOKE_REQUEST}, mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, "model": "claude-opus-5-5", - "id": "msg_bdrk_mxljoibckai4wqyt4wd7cmslqjtlkals5cbwms2hackerasjglha", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_qyqgstng2m3crzmaqzbueka3ep22c7lsslqm5vqoatdm7pi37crq", + "content": [ + { + "type": "thinking", + "thinking": "I need to check the validation file, particularly around line 18 where the user wants me " + "to look.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` around `validate_quantity` (line 18) to see how zero, " + "negative and non-integer values are checked.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_016r8yEaJbeHsNj69b8tCYY5", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 45}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 163, + "output_tokens_details": {"thinking_tokens": 18}, "service_tier": "standard", }, }, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "bedrock/invoke/us.anthropic.claude-opus-5-5", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", + "content": "I'll read `shop/validation.py` around `validate_quantity` (line 18) to see how zero, " + "negative and non-integer values are checked.", "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 45}', + "name": "read_file", + }, + "id": "toolu_bdrk_016r8yEaJbeHsNj69b8tCYY5", + "type": "function", + }, + ], + "reasoning_content": "I need to check the validation file, particularly around line 18 where the user wants " + "me to look.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I need to check the validation file, particularly around line 18 where the " + "user wants me to look.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I need to check the validation file, particularly around line 18 where the " + "user wants me to look.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + }, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 163, + "prompt_tokens": 880, + "total_tokens": 1043, + "completion_tokens_details": {"reasoning_tokens": 18, "text_tokens": 145}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, "service_tier": "standard", }, }, ) +CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0: Final = ( + "CAQS5AcKEggSEAEYAjgBQgh0aGlua2luZxIMZ3CC5ydK6won2yd9GgxE4gMZ+E/WBfdMZvAiMEVQQ4qw23oAzYzAw0ZKeKq2slCfdKWoUJRd" + "3yCnR0UZgiVPAHOYvnJkBTUpMEzpOSr/BuPzDT9Eso9jzyJX/huQF3Pd7Hwlj+PQFM9uKARSgoG2Dsr1igyT1mEfbp/ePlaLZdxWFpHnHjDk" + "IiVcVqpUTsJ9XSCQyYAOENmalP+NP+DJ2JkQyaZNf3DIIdDNpOWT1hhq2CdYoHu0yuyUTbeihla45+rF68EY3TplHFoDAatrii0itWJ+cA5b" + "OHdolQY1UUyVFkhWyxmiOlh0Zt2Tc/ITerLKB0fySyHWmBzwdU27u5Q1kCeOFB/Qx0FMz8X/tWz034Lh17vJsrxud7PNvREUuYfpDRFrr9+l" + "HEXP2H/BqVEsN55egf1Nc8vu5E5EmBAxU3B7gkJes+PiD/yj/LkwUPDlUZGJnKJB7Vt1ylYcxHaMAfavUHrBLsTbvx6lwyor66phw6v6lQNR" + "TDsMY6qibpE+H5bY5bpGtdH772Z9LWyHRXG94qNHg6YjH46m/NlocjSk5C4U9i6YqYAPYZrHFO4ELzwuqaNmWXUuf4FoL70DAUeUNPn4dsJl" + "lKWyrqs/LmrGolJsde8GpwpTlNNRMkm1hhBMpCnjSrzCS9GElJ0FjF4zrzKpcFmVqGYJ7jf1mWA0lnGmUJQK8FPTAIhtQDgSWmh6sDos4A1T" + "ox12fHGTxflSqi3CIF57+NFmLefxTJLgOEmY8i8yDssZrnDsJC2/1RoKxgEfUJ7a1S6Tx9LWLVwDJJU81FJkCk3R8ZIc3boT7wzlm5Nmxb02" + "r/NubRSPOdsL5tL4UltVQJv2KMGK2/gIgF5vFB5LM5LtBNtdfLolEd6Nar0Loz8aI11mZ0toCIlo8m9nLdut9wx1cjB4ZqmQQLgaZIiM8ZuD" + "I07fPCnfgsnSWAdJ2TrPioGSv1Z0wz+vMZMYzB1ebNpl66WlpWJPTVwbjc7YZH7gNKxduopuh6ZGikhtXuCsv7Xm3K4jb8mRW0esr43IjGO7" + "u4U6bcDAOUwqBa4vR3czuHPSHn2EI69WlPEP1G4Im+DS1G25SJ6VVj4WkYmmid+OEuamPWNYgQ/RUL2nOBSzLFhfBl/R8Dk3ozTOIcooqAB6" + "9e1vCyVh27iTrRahadf0Zx9EqYjc6anG/lIerWRiFvMhZBjvdSbU6H7Ft54ACQUq4ZVGJt747CgmtS2gmokM8klKqkeXZPxlIcPd6yJvxGa4" + "MCh929kyGTs1EkoAt8Qu4OR924AtF3KkBQinwisYAQ==" +) CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-fable-5-1", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-fable-5-1"}, expected_provider_endpoint="/model/us.anthropic.claude-fable-5-1/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - }, + expected_provider_request={**BEDROCK_INVOKE_REQUEST}, mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_bdrk_g7xqazu6324fsulzlcgyoh4ixc2pytulkxmgzcjh6bvesncpiweq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_ygn75gjwiywsmw52hhhtugcac2xa3b6qjaoygnm3ihch23zsv4ba", + "content": [ + { + "type": "thinking", + "thinking": "The user explicitly said to read the validation file first, so I'll respect that " + "sequencing rather than reading both files in parallel. I'll read the validation file now.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` around `validate_quantity` to see how zero and negative " + "values are checked.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01MGYCujwiQim9cJXKY33qzc", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 276, + "output_tokens_details": {"thinking_tokens": 141}, "service_tier": "standard", }, }, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "bedrock/invoke/us.anthropic.claude-fable-5-1", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", + "content": "I'll read `shop/validation.py` around `validate_quantity` to see how zero and negative " + "values are checked.", "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "name": "read_file", + }, + "id": "toolu_bdrk_01MGYCujwiQim9cJXKY33qzc", + "type": "function", + }, + ], + "reasoning_content": "The user explicitly said to read the validation file first, so I'll respect that " + "sequencing rather than reading both files in parallel. I'll read the validation file " + "now.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user explicitly said to read the validation file first, so I'll respect " + "that sequencing rather than reading both files in parallel. I'll read the " + "validation file now.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user explicitly said to read the validation file first, so I'll " + "respect that sequencing rather than reading both files in parallel. I'll " + "read the validation file now.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + ], + }, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 276, + "prompt_tokens": 880, + "total_tokens": 1156, + "completion_tokens_details": {"reasoning_tokens": 141, "text_tokens": 135}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, "service_tier": "standard", }, diff --git a/tests/integration/translation/chat_completions/bases/bedrock_mantle.py b/tests/integration/translation/chat_completions/bases/bedrock_mantle.py index 458f0834a86..1450ec01a8c 100644 --- a/tests/integration/translation/chat_completions/bases/bedrock_mantle.py +++ b/tests/integration/translation/chat_completions/bases/bedrock_mantle.py @@ -2,7 +2,14 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase - +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.bedrock_mantle import ( + BEDROCK_MANTLE_ANTHROPIC_MESSAGES_REQUEST, + BEDROCK_MANTLE_ANTHROPIC_MESSAGES_RESPONSE, + BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, +) CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -163,86 +170,6 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( ) -CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock_mantle/anthropic.claude-opus-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/anthropic/v1/messages", - expected_provider_headers={ - "content-type": "application/json", - "authorization": "Bearer synthetic-bedrock-mantle-key", - }, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - "model": "anthropic.claude-opus-5-5", - }, - mock_provider_response={ - "model": "claude-opus-5-5", - "id": "msg_bdrk_ynrmln5e2d6gulngkuay6eypqvjtghpnmtiioovu57numo6rag2q", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - }, - }, - expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "bedrock_mantle/anthropic.claude-opus-5-5", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, - } - ], - "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "service_tier": "standard", - }, - }, -) - - GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", @@ -533,100 +460,76 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) - -GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock_mantle/openai.gpt-6-luna", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/v1/responses", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock_mantle/anthropic.claude-opus-5-5"}, + expected_provider_endpoint="/anthropic/v1/messages", expected_provider_headers={ - "authorization": "Bearer synthetic-bedrock-mantle-key", "content-type": "application/json", + "authorization": "Bearer synthetic-bedrock-mantle-key", }, - expected_provider_request={ - "model": "openai.gpt-6-luna", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "stream": False, - }, - mock_provider_response={ - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791240609, - "created_at": 1791240609, - "error": None, - "frequency_penalty": 0.0, - "id": "resp_y6jkekxfsn43teacrqd745fnw5m6l3q6sczasxu6y4nxmaoittxq", - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, - "metadata": {}, - "model": "openai.gpt-6-luna", - "object": "response", - "output": [ - { - "content": [{"annotations": [], "logprobs": [], "text": "Hello!", "type": "output_text"}], - "id": "msg_fb51254d3c005892a700f8dc8c55b04d", - "phase": "final_answer", - "role": "assistant", - "status": "completed", - "type": "message", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "in_memory", - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "safety_identifier": None, - "service_tier": "default", - "status": "completed", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", - "usage": { - "input_tokens": 19, - "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, - }, - "user": None, - "moderation": None, - }, + expected_provider_request={**BEDROCK_MANTLE_ANTHROPIC_MESSAGES_REQUEST}, + mock_provider_response={**BEDROCK_MANTLE_ANTHROPIC_MESSAGES_RESPONSE}, expected_litellm_response={ - "id": "resp_y6jkekxfsn43teacrqd745fnw5m6l3q6sczasxu6y4nxmaoittxq", - "created": ANY, - "model": "bedrock_mantle/openai.gpt-6-luna", - "object": "chat.completion", - "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello!", "role": "assistant"}}], + **CHAT_COMPLETIONS_RESPONSE, + "model": "bedrock_mantle/anthropic.claude-opus-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "Next I'll read `shop/validation.py` around `validate_quantity` (line 18) to see how it " + "handles zero, negative and non-integer values.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "name": "read_file", + }, + "id": "toolu_bdrk_018J8SJwAgjhRkkmtm3xdJx2", + "type": "function", + }, + ], + "reasoning_content": "I should check the validation file first, starting around line 18.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I should check the validation file first, starting around line 18.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I should check the validation file first, starting around line 18.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + }, + }, + }, + ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 19, - "total_tokens": 25, - "completion_tokens_details": {"reasoning_tokens": 0}, - "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, + "completion_tokens": 166, + "prompt_tokens": 880, + "total_tokens": 1046, + "completion_tokens_details": {"reasoning_tokens": 19, "text_tokens": 147}, + "prompt_tokens_details": { + "cached_tokens": 876, + "text_tokens": 4, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "service_tier": "standard", }, - "billing": {"payer": "developer"}, - "frequency_penalty": 0.0, - "presence_penalty": 0.0, - "service_tier": "default", }, ) @@ -634,90 +537,385 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "bedrock_mantle/openai.gpt-6.1-sol", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock_mantle/openai.gpt-6.1-sol"}, expected_provider_endpoint="/openai/v1/responses", expected_provider_headers={ "authorization": "Bearer synthetic-bedrock-mantle-key", "content-type": "application/json", }, expected_provider_request={ - "model": "openai.gpt-6.1-sol", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, "stream": False, }, - mock_provider_response={ - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791240612, - "created_at": 1791240612, - "error": None, - "frequency_penalty": 0.0, - "id": "resp_rjew7tdxw2lzv5egpmamwt2g5aifjxopahkpmzawjlcyucblokyq", - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, - "metadata": {}, - "model": "openai.gpt-6.1-sol", - "object": "response", - "output": [ - { - "content": [{"annotations": [], "logprobs": [], "text": "Hello!", "type": "output_text"}], - "id": "msg_6f7c19b0241956fda392424fc5aa3db9", - "phase": "final_answer", - "role": "assistant", - "status": "completed", - "type": "message", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "in_memory", - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "safety_identifier": None, - "service_tier": "default", - "status": "completed", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", - "usage": { - "input_tokens": 19, - "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, - }, - "user": None, - "moderation": None, - }, + mock_provider_response={**BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE}, expected_litellm_response={ - "id": "resp_rjew7tdxw2lzv5egpmamwt2g5aifjxopahkpmzawjlcyucblokyq", - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_izakuft4jrxwmodefqee2jjt6gkmayywwuuc242pduoydywb3v4a", "model": "bedrock_mantle/openai.gpt-6.1-sol", - "object": "chat.completion", - "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello!", "role": "assistant"}}], + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I’ll read `shop/validation.py` first to see how it handles zero, negative, and " + "non-integer quantities before inspecting the tests.", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_399ba0e7ac865a798b46a6cabe6b6bde", + "type": "function", + }, + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_7a90cfc98ff653aeb165c2b845bc1e4b", + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJ7rFK0vbfOLCW3lZUKZX7nZu5wsyKm4u2DeoLRbILW0AF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBN1N2LzZrWDd1QWV1WkpjQ1dEOFQvSDR2Y0ZaY2ZWemtpR2ZWNGRncEgwZmJqMVBNd3FIV2NnbFlTcHVOcVMyZWc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcmpoCBv5IPHctwun5UTXrXrEmFVDe7MUULd6PZATyexPQO0PepR8jRQ3zVYSjkLkI8eYSjtPsT9kCKrIpsJlAuid+IimJvnTBHGplsaTC/53IpOSzB87nJlx+m4kCAAAQANtzzELZF5mKNBTvW/ca/WZ6iJI42OwsDigg6J5ZOIdJaC35TDfpQsssoDFQIHtI/v////8AAAABAAAAAAAAAAAAAAABAAABSyq/hcdJ8rImbQvE2c8qmEFM3skJI4sMoBBIdIZm8ljKX6zuu9kBGRln5StGWzlUOVYemoWeHBoDDuZkGDWAeTcQYdNeiCuTOQBg1kfIvhSGpGZHyKIfg+XnWIjrExKbZhmAC/GMebMNNuJ1/ZUFnq0UtD8X1r2xEkIWCUsQHysv9E6D6AstjrAgpo5arvZG0Gi5gryi/eE9Q7IGwilIvzeslUlM7sVcmD5RWe83nMeXmCUR41N50TK11F1j/H314CtKbJbMQWNWGLUUSRiKjqDFiC+o4bioQCRzvka6NUkIBitgjjkJ+etpL5ibB065xADWxaRc6WoUe1VkNVKXlYq4kOBNhnkJSHuasIJiD8/suYWXshD9XMF+SffKyKk3zscZwAuZ7N/ZOayYqGrhOMryCFYXvSo7v8PJ1hvg64z1MknImFG/6rSEVox9yeyfSEQmeDX/ABwO8PAsAGcwZQIwIpngWR/1wARV01RTUI+B4dV7XaR1AC0aOv5KfuQw5UJOnfGVp8E7MS379SPiA+SEAjEA0TMJ1LJ3cXitGqWvPATglnscU7UW9Mhny4elQ+cw7pFm5kRlYA/lkAyDdNyjZFVU", + "summary": [], + }, + ], + }, + }, + ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 19, - "total_tokens": 25, - "completion_tokens_details": {"reasoning_tokens": 0}, + "completion_tokens": 108, + "prompt_tokens": 312, + "total_tokens": 420, + "completion_tokens_details": {"reasoning_tokens": 39}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, + }, + "billing": {"payer": "developer"}, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "service_tier": "default", + }, +) + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock_mantle/openai.gpt-6-luna"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-6-luna", + "stream": False, + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027279, + "created_at": 1792027278, + "id": "resp_pbro2cu7wxoshets5f2v3mmv22dzjjepi3ztmm6qceyuuveilumq", + "model": "openai.gpt-6-luna", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeO/Kvntq5madicvQnPlNuvNXUggyOe9VUAXpEAJIioejAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBd3VOZ05NSUlPMW5NbkYrOTRMbVdXb2tXcEpDRE9VaUl0eFF0UzFtUjNBUGlvcUVIWlNlYm1TMFZDQmlqdEJ5WkE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBczj0IBqgjEgu5ZgKVdTj3OPcpzd2fIh34hEYxgwTyexPQO0PepR8jRQ3zVYTReEQNipLtt5xyBLb3uYPRCtqi6ry4MIerha2orIM/kNwSud8QD55iZRsgKj/1XbcCAAAQAJdjRdvVBzuOT4dEabwB/y7OUCvt4aMTO2tMpab00xmq/PMDdaAtcFiOCt2np+72If////8AAAABAAAAAAAAAAAAAAABAAABU4/yBtf3Lh0ztBhEM9+zVRuLBf3cQi+TBEvKzqKrCRcFAt+D762ck8n+qzDT/Kun8V8tLn5f1CQMMOdn2cu7aDV+4yYud8rII/NzEYzht+XQy0doyMj4GLfIW8jJ81IBh02MAaK+b1dZilKsA6448TsvE+58lIPPnJ7vScz2sRQzotxYqb0EIj6V+zl980Ucs+Ge1SGRwMVeUj/5mEspkmtJt0Tjkyku2jdqokDqpDXFBQDLAIKv+sRfpO6BQL64bovGzIhP+FSo1QFPW+NSpfILrWhB8WwUzRnoqrKcSTmqkDAx7v/wzUWeN0rl1XjbQgcJDs4kjgcBTR/4nhVX7rlAmiDaeZkZHDTKYYq67BaAwF35TuWAfz3cvusF1FZ6CQETRnkXXfRqQITDG6CEADQDrUtp+8kF5vIsFGX3ivYEDtlrKRcriWlq30dHn2ewRh8B1Z+k+GheeEyoDt3SC2KhsvQAZzBlAjAENp2ZItMXMogj4+bqZ1DFA/VBQb4fEwuIJlAEXCi2NWbzA7GFtwVfyqcx3nGPs8ICMQDPeXyziTwCViPMplaixJa2Z+6m8FsynJE6Gt4joeNS1Gm36RHhrdmv7Nx68QTw7RA=", + "id": "rs_41fd1f946fd95402ac51feb270547528", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I need `shop/validation.py` next; it contains the quantity validator, so it will " + "show how zero, negative, and non-integer values are handled.", + "type": "output_text", + }, + ], + "id": "msg_bd4b22ee05d75ecc863a79cadb0abc97", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":100}', + "call_id": "call_2bb9788b89cf56e595d03edd897cf96d", + "id": "fc_6cec15107b1055dd90bdbf343add57e6", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "usage": { + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 115, + "output_tokens_details": {"reasoning_tokens": 42}, + "total_tokens": 427, + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_pbro2cu7wxoshets5f2v3mmv22dzjjepi3ztmm6qceyuuveilumq", + "model": "bedrock_mantle/openai.gpt-6-luna", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I need `shop/validation.py` next; it contains the quantity validator, so it will show " + "how zero, negative, and non-integer values are handled.", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":100}', + "name": "read_file", + }, + "id": "call_2bb9788b89cf56e595d03edd897cf96d", + "type": "function", + }, + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_41fd1f946fd95402ac51feb270547528", + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeO/Kvntq5madicvQnPlNuvNXUggyOe9VUAXpEAJIioejAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBd3VOZ05NSUlPMW5NbkYrOTRMbVdXb2tXcEpDRE9VaUl0eFF0UzFtUjNBUGlvcUVIWlNlYm1TMFZDQmlqdEJ5WkE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBczj0IBqgjEgu5ZgKVdTj3OPcpzd2fIh34hEYxgwTyexPQO0PepR8jRQ3zVYTReEQNipLtt5xyBLb3uYPRCtqi6ry4MIerha2orIM/kNwSud8QD55iZRsgKj/1XbcCAAAQAJdjRdvVBzuOT4dEabwB/y7OUCvt4aMTO2tMpab00xmq/PMDdaAtcFiOCt2np+72If////8AAAABAAAAAAAAAAAAAAABAAABU4/yBtf3Lh0ztBhEM9+zVRuLBf3cQi+TBEvKzqKrCRcFAt+D762ck8n+qzDT/Kun8V8tLn5f1CQMMOdn2cu7aDV+4yYud8rII/NzEYzht+XQy0doyMj4GLfIW8jJ81IBh02MAaK+b1dZilKsA6448TsvE+58lIPPnJ7vScz2sRQzotxYqb0EIj6V+zl980Ucs+Ge1SGRwMVeUj/5mEspkmtJt0Tjkyku2jdqokDqpDXFBQDLAIKv+sRfpO6BQL64bovGzIhP+FSo1QFPW+NSpfILrWhB8WwUzRnoqrKcSTmqkDAx7v/wzUWeN0rl1XjbQgcJDs4kjgcBTR/4nhVX7rlAmiDaeZkZHDTKYYq67BaAwF35TuWAfz3cvusF1FZ6CQETRnkXXfRqQITDG6CEADQDrUtp+8kF5vIsFGX3ivYEDtlrKRcriWlq30dHn2ewRh8B1Z+k+GheeEyoDt3SC2KhsvQAZzBlAjAENp2ZItMXMogj4+bqZ1DFA/VBQb4fEwuIJlAEXCi2NWbzA7GFtwVfyqcx3nGPs8ICMQDPeXyziTwCViPMplaixJa2Z+6m8FsynJE6Gt4joeNS1Gm36RHhrdmv7Nx68QTw7RA=", + "summary": [], + }, + ], + }, + }, + ], + "usage": { + "completion_tokens": 115, + "prompt_tokens": 312, + "total_tokens": 427, + "completion_tokens_details": {"reasoning_tokens": 42}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, + }, + "billing": {"payer": "developer"}, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "service_tier": "default", + }, +) + +GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock_mantle/openai.gpt-6-astra"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-6-astra", + "stream": False, + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027285, + "created_at": 1792027282, + "id": "resp_jpvjizqjx4f4u6nyz7deamnfdzbqxgxqm7nirzdlaxm2mbp5r7yq", + "model": "openai.gpt-6-astra", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeOrR57yIGw3WXzOCHxReanHc4wZJmaqSvXVqP0qGFGOIAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBK1FoNHBXMk1DRU5Vdis5d3M3OVBTVDBuU09VTkhlUDViMDkwejZPeWZpOEhOWnBjYXNKNmJVL0J5VWxHZDA2bFE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcM8fQN00KGzoCPxlReE9xaFkOBsPUkSLsg/xl/wTyexPQO0PepR8jRQ3zVYQk8QfLD0eucuatvVchgKNLGMVoDPckkpY6SgFElbU1FqyUMQRvNUXBscq6GwAQezoCAAAQAGSi7URwBzZfrKqE7cF8zQKcbSay4uxRdsBmjsgYzhEssEjRy/XHvWoc05wPhp6Fb/////8AAAABAAAAAAAAAAAAAAABAAABsukDvrtRmG7Y5bL3xQiRmAPLsY2Wf14e5hnSAPYx/GMaB6IatF4KG1RSZK14/tlh+/kB5vAcOrXrllEzvZHIFDMWgRWexPSGO/WTBZ8GbLNmr5CZo9bMu0wxt86lxm+clENYV8HgEIZC7Gg8gQLw1IgT97PiOaYqg63/uHN6VME19CtW68lifkgtupCAP9ewU0/W1BG9TggARvgfL6Rhn087DjY8F0ievrVLHQRujVnEZa+Yfsk+WVCT156LYo2wGLwkcZDycPX7/2Qn6hiHm7/vUq2OYGSWDOabutgooMUhX/vA+0u3cEHQu+Vv0QbwaUH5+MLB0L85bX58U3HlF6KG02vuGl8TiI74zvzYx00YFoP+n6NmaH2VoY8OaaTr6hQKEpHqiesAPWXdAA1+TmJ9mMjTATYyxdsj7E3X0+djoKENYqaNqA7tlEdm9EvKnMPtHWtsSUsO4nB38nsQyHKJqyUQP/MauHrq+YDx4GKwB45EnvGADOCRf7uuub7mACOhersjo1T7SGCTq9OP73x7/tYtSKeYVGZlMMAJGTmc2o/l982eGRf6B+/o605lqvCYX3ZBTuAr+823Wx/1uuvLNgBnMGUCMQDrd4nbQJvNcl/n8tW+ibBjExgdHHqO14vyB+oBDToTD5XeX/nI8UZKrE2aFLHsXo8CMBaXARx3KSuVYzI8Dsqn5Br0DPqZx5n7NmRNhqUvL0y4ZhMpivrpq1xc2uW3vUyYxQ==", + "id": "rs_5ddaaaa907d7599da226348b310ffaa1", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, " + "and non-integer quantities.\n", + "type": "output_text", + }, + ], + "id": "msg_7ee57d84b2515199bcfd28428434f307", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_48b391faab505f0e90dfc7c451b87c2b", + "id": "fc_c8b90cef33d256a986e7a9229db5017a", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "usage": { + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 148, + "output_tokens_details": {"reasoning_tokens": 83}, + "total_tokens": 460, + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_jpvjizqjx4f4u6nyz7deamnfdzbqxgxqm7nirzdlaxm2mbp5r7yq", + "model": "bedrock_mantle/openai.gpt-6-astra", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_48b391faab505f0e90dfc7c451b87c2b", + "type": "function", + }, + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_5ddaaaa907d7599da226348b310ffaa1", + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeOrR57yIGw3WXzOCHxReanHc4wZJmaqSvXVqP0qGFGOIAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBK1FoNHBXMk1DRU5Vdis5d3M3OVBTVDBuU09VTkhlUDViMDkwejZPeWZpOEhOWnBjYXNKNmJVL0J5VWxHZDA2bFE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcM8fQN00KGzoCPxlReE9xaFkOBsPUkSLsg/xl/wTyexPQO0PepR8jRQ3zVYQk8QfLD0eucuatvVchgKNLGMVoDPckkpY6SgFElbU1FqyUMQRvNUXBscq6GwAQezoCAAAQAGSi7URwBzZfrKqE7cF8zQKcbSay4uxRdsBmjsgYzhEssEjRy/XHvWoc05wPhp6Fb/////8AAAABAAAAAAAAAAAAAAABAAABsukDvrtRmG7Y5bL3xQiRmAPLsY2Wf14e5hnSAPYx/GMaB6IatF4KG1RSZK14/tlh+/kB5vAcOrXrllEzvZHIFDMWgRWexPSGO/WTBZ8GbLNmr5CZo9bMu0wxt86lxm+clENYV8HgEIZC7Gg8gQLw1IgT97PiOaYqg63/uHN6VME19CtW68lifkgtupCAP9ewU0/W1BG9TggARvgfL6Rhn087DjY8F0ievrVLHQRujVnEZa+Yfsk+WVCT156LYo2wGLwkcZDycPX7/2Qn6hiHm7/vUq2OYGSWDOabutgooMUhX/vA+0u3cEHQu+Vv0QbwaUH5+MLB0L85bX58U3HlF6KG02vuGl8TiI74zvzYx00YFoP+n6NmaH2VoY8OaaTr6hQKEpHqiesAPWXdAA1+TmJ9mMjTATYyxdsj7E3X0+djoKENYqaNqA7tlEdm9EvKnMPtHWtsSUsO4nB38nsQyHKJqyUQP/MauHrq+YDx4GKwB45EnvGADOCRf7uuub7mACOhersjo1T7SGCTq9OP73x7/tYtSKeYVGZlMMAJGTmc2o/l982eGRf6B+/o605lqvCYX3ZBTuAr+823Wx/1uuvLNgBnMGUCMQDrd4nbQJvNcl/n8tW+ibBjExgdHHqO14vyB+oBDToTD5XeX/nI8UZKrE2aFLHsXo8CMBaXARx3KSuVYzI8Dsqn5Br0DPqZx5n7NmRNhqUvL0y4ZhMpivrpq1xc2uW3vUyYxQ==", + "summary": [], + }, + ], + }, + }, + ], + "usage": { + "completion_tokens": 148, + "prompt_tokens": 312, + "total_tokens": 460, + "completion_tokens_details": {"reasoning_tokens": 83}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, + }, + "billing": {"payer": "developer"}, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "service_tier": "default", + }, +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "bedrock_mantle/openai.gpt-5.6-terra"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-5.6-terra", + "stream": False, + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027294, + "created_at": 1792027292, + "id": "resp_rzoladobotq7d5aitvspg3wwysoow5b436v6zaymozikilgh4uwa", + "model": "openai.gpt-5.6-terra", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJSjZG0YBxqkNyCzhib06j32zRnkxAVgKyx5FlM5EQsGAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBblF2WjlWWElaWDBLWXR0RDE3djIxTzd2bWJVdjZSQlhOYW1sMmxjTHRrVGFuVU4vc09LRDJlUzFGMENIQzdOa0E9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcL/APOsU1Boid3k+fiCN6FGnqlouHrRGycfpWSQTyexPQO0PepR8jRQ3zVYQqZPsD7v2uGJDXTNoMQ/DNJEPAmyJkLXtVRDtxK+LIpvBebVGlBLXtrij6Z/oBFI8CAAAQABUJCnt2FvwOmW31K7V/wAUU+CK5nMfo/9o1zvOTBV0aDdylXF3v56EtlCEoo8vw6/////8AAAABAAAAAAAAAAAAAAABAAABPxjRAPZm81KRyzjmvtQM3zhh5qZSePYBGrVVjgFK2e9hY/yVx8S8S70YklxeYaWmFkKyinCLXqUH0IFnsyUaBQGpvnrVixo+QDyeu4wV6WP5hyp7j/GGWaAlDM3zF4JkOIqpXYU2XbHwr1/wRzimQl3BcBxBpygEBHteEuAWENg16dfo76pI4Q+PpGOKfwaSXz9Cj2vprUCxGcsa9IY6tq6pr7hYdhtS4Zg8tYGrSoGm6/9ljF51LZ1TSvfHnKmNFgpp86naM08zO1Jp+fDU37fKH8nazA0xUB23BWMWWu63pvQaquAtx1rj5PisEWwemlY/5v/Wo4d4Q9W+nrGzXf2o9kScWnDpfCfR7G72u19J0SXHcfrFwj5oVN8Ek1eYKY8Apv2lxZAf+ULokPmUGk/lXkz9hxa52txJ/3+bjy7cJ4S9HoUuBRHt3l7PpJJ0AGcwZQIxAP/SkcKkldJ07XgfZ+neD7X9h8kLD3hHG4axXwP8LPDmZ6HKrnXbOoSZLT6Nf4JddAIwEte72+56IOzdbE/k0Zvq/5ILQB5G+FhUdu0tHd6YDhHqPuq4nzXJJ/+0dFHRWzcw", + "id": "rs_2985de4698f95e958cd7c7cd377a76a2", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "Next I need `shop/validation.py`, which contains `validate_quantity`; I’ll " + "inspect it to verify the handling of zero, negative, and non-integer inputs before " + "reviewing the tests.", + "type": "output_text", + }, + ], + "id": "msg_afefa5c808a159a1965dc7c7986d0693", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_fc5f6bc78e0b5332b7866a98d5470b46", + "id": "fc_1e6d0bcf394d5b64badb08659c4c6028", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "usage": { + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 114, + "output_tokens_details": {"reasoning_tokens": 36}, + "total_tokens": 426, + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "id": "resp_rzoladobotq7d5aitvspg3wwysoow5b436v6zaymozikilgh4uwa", + "model": "bedrock_mantle/openai.gpt-5.6-terra", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "Next I need `shop/validation.py`, which contains `validate_quantity`; I’ll " + "inspect it to verify the handling of zero, negative, and non-integer inputs before " + "reviewing the tests.", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_fc5f6bc78e0b5332b7866a98d5470b46", + "type": "function", + }, + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_2985de4698f95e958cd7c7cd377a76a2", + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJSjZG0YBxqkNyCzhib06j32zRnkxAVgKyx5FlM5EQsGAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBblF2WjlWWElaWDBLWXR0RDE3djIxTzd2bWJVdjZSQlhOYW1sMmxjTHRrVGFuVU4vc09LRDJlUzFGMENIQzdOa0E9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcL/APOsU1Boid3k+fiCN6FGnqlouHrRGycfpWSQTyexPQO0PepR8jRQ3zVYQqZPsD7v2uGJDXTNoMQ/DNJEPAmyJkLXtVRDtxK+LIpvBebVGlBLXtrij6Z/oBFI8CAAAQABUJCnt2FvwOmW31K7V/wAUU+CK5nMfo/9o1zvOTBV0aDdylXF3v56EtlCEoo8vw6/////8AAAABAAAAAAAAAAAAAAABAAABPxjRAPZm81KRyzjmvtQM3zhh5qZSePYBGrVVjgFK2e9hY/yVx8S8S70YklxeYaWmFkKyinCLXqUH0IFnsyUaBQGpvnrVixo+QDyeu4wV6WP5hyp7j/GGWaAlDM3zF4JkOIqpXYU2XbHwr1/wRzimQl3BcBxBpygEBHteEuAWENg16dfo76pI4Q+PpGOKfwaSXz9Cj2vprUCxGcsa9IY6tq6pr7hYdhtS4Zg8tYGrSoGm6/9ljF51LZ1TSvfHnKmNFgpp86naM08zO1Jp+fDU37fKH8nazA0xUB23BWMWWu63pvQaquAtx1rj5PisEWwemlY/5v/Wo4d4Q9W+nrGzXf2o9kScWnDpfCfR7G72u19J0SXHcfrFwj5oVN8Ek1eYKY8Apv2lxZAf+ULokPmUGk/lXkz9hxa52txJ/3+bjy7cJ4S9HoUuBRHt3l7PpJJ0AGcwZQIxAP/SkcKkldJ07XgfZ+neD7X9h8kLD3hHG4axXwP8LPDmZ6HKrnXbOoSZLT6Nf4JddAIwEte72+56IOzdbE/k0Zvq/5ILQB5G+FhUdu0tHd6YDhHqPuq4nzXJJ/+0dFHRWzcw", + "summary": [], + }, + ], + }, + }, + ], + "usage": { + "completion_tokens": 114, + "prompt_tokens": 312, + "total_tokens": 426, + "completion_tokens_details": {"reasoning_tokens": 36}, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, "billing": {"payer": "developer"}, diff --git a/tests/integration/translation/chat_completions/bases/gemini.py b/tests/integration/translation/chat_completions/bases/gemini.py index 1ffde32c3be..915894312aa 100644 --- a/tests/integration/translation/chat_completions/bases/gemini.py +++ b/tests/integration/translation/chat_completions/bases/gemini.py @@ -1,7 +1,396 @@ +from collections.abc import Mapping from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.gemini import ( + GEMINI_3_8_FLASH_THOUGHT, + GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + GEMINI_GENERATE_CONTENT_REQUEST, + GEMINI_GENERATE_CONTENT_RESPONSE, +) +from pydantic import JsonValue + +GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "gemini/gemini-3.8-flash"}, + expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request=GEMINI_GENERATE_CONTENT_REQUEST, + mock_provider_response=GEMINI_GENERATE_CONTENT_RESPONSE, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "gemini/gemini-3.8-flash", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": ( + "To check how zero, negative, and non-integer quantities (such as floats, non-numeric values, or " + "values less than or equal to zero) are validated, I need to inspect `shop/validation.py` next." + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + }, + "function": { + "arguments": '{"path": "shop/validation.py", "end_line": 50, "start_line": 1}', + "name": "read_file", + }, + "id": "call_7668__thought__" + GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + "type": "function", + }, + ], + "images": [], + "reasoning_content": GEMINI_3_8_FLASH_THOUGHT, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": GEMINI_3_8_FLASH_THOUGHT, + }, + ], + "provider_specific_fields": { + "thought_signatures": [ + GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + ], + }, + }, + "provider_specific_fields": { + "native_finish_reason": "STOP", + }, + }, + ], + "usage": { + "completion_tokens": 655, + "prompt_tokens": 358, + "total_tokens": 1013, + "completion_tokens_details": { + "reasoning_tokens": 575, + "text_tokens": 80, + }, + "prompt_tokens_details": { + "text_tokens": 358, + }, + }, + "vertex_ai_citation_metadata": [], + "vertex_ai_grounding_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_url_context_metadata": [], + }, +) + +GEMINI_3_1_PRO_PREVIEW_THOUGHT: Final = ( + "**Analyzing Quantity Validation in shop/validation.py**\n\n" + "Okay, so the goal here is pretty clear: the user wants to understand how the system validates th" + "e `quantity` field, specifically how zero and negative quantities are handled. My job is to fin" + "d the relevant code and explain what's happening.\n\n" + "The search results pointed to `shop/validation.py`, and specifically to a `validate_quantity` fu" + "nction on line 18. Excellent! That's a direct lead.\n\n" + "Now, I need to get a look at that code. To do that, I'm going to leverage the `read_file` tool. " + "I'm thinking I'll read a block of code around that line – let's say from line 10 to 30 – to give" + " us some context around the function itself. That should give me enough to validate our underst" + "anding of the user request.\n\nIn summary, I'm going to read the `shop/validation.py` file to exam" + "ine the `validate_quantity` function, then I will call the tool.\n\n\n" +) + +GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = ( + "ErAFCq0FAWkUfRNlFv/lCrWnTeRSJUQxveeFy7unSAIXuMtxRKgoOGMtvJFe+qjPDFHdUuZP3JdL5ppGcmh8B8rxQ2ixle8Z" + "4kyuY1ZCCxoXUYj4al2xk6tKOKmB9Nwcyk8EMfl34HkY4wwzPdnHd5LiHDpdOG+LRDtvCTn0sIZ5vp7xEba4z/3OoSJkNZ5Y" + "dhprkDwNqSEJrCujrSwejQlRBarfIBzKmIMsOuOVuX+jTOZjLjuY2BQ+WaFBze/Nkl3lqBEs1ItCelts/L+6SAFvuzeCdcrt" + "ahBH6+t8M9ZPOCkWcvrCiuwi8xF3AzAhTdTfEcECHFB4HpiJbuPRGNWVNO+c0Xj1tti+oOM6hamlxxiNiE6qEu6fVGI/f14u" + "9fPL5XpGla06CtB5YIo94WifNUdpaF1ydl1VNs7N/qJwH0glZmTqiNGuP3SyURtRg/S7TTIHp2dMyTYWc/baAl0p6g5vJuHd" + "6vN3yBOa8jnUS7di1WKamCFnqKck8vmfXHGwiWsUVHn2SKrI/PKoWOTTw3uc3+Z37pRl/+L9SpNdbfecvOqTrqw12iVi38QY" + "HEukvJu6FqlRQOvOt3wPbb/RAVStD0fpRTQnTkYrS/lI2inNGrDWWpPpi/0u7dWrzJFOGO/7jV+63BggBPplEVcEkCmZrrZC" + "2XAsnjzTzAhPzgZ8kR0m0pvjq2lbJgREIeai4ygQ6q0OQkclZmQChISjtw1K/lSuuNlJCrwZFn9Xsp6JaJ6GrGkaJsbgCOOH" + "bsDyGOba58qNXvvHq33A0km8GthfqiEoEwxLcHwyH3jDDnuYf+/XdqdckdXcMf1T9HcMF1xMhYdkjlYSgVP5iRMIJ/2R1WgK" + "gXoTgXjLCfHUnUUjFFjpgJwEzqfd8M+7cJfS9tcK07Pu4FSmuZcbCuDX9w==" +) + +GEMINI_3_1_PRO_PREVIEW_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + "thought": True, + }, + { + "text": ( + "I will now read the `shop/validation.py` file to see the implementation of the `validate_quantit" + "y` function." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "start_line": 10, + "end_line": 30, + }, + "id": "call_1654638", + }, + "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 60, + "totalTokenCount": 568, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 150, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "2ybQaqy6Gd2i6dkPm82G2AU", +} + +GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "gemini/gemini-3.1-pro-preview"}, + expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request=GEMINI_GENERATE_CONTENT_REQUEST, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_3_1_PRO_PREVIEW_RESPONSE_FIELDS}, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "gemini/gemini-3.1-pro-preview", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": ( + "I will now read the `shop/validation.py` file to see the implementation of the `validate_quantit" + "y` function." + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + }, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 10, "end_line": 30}', + "name": "read_file", + }, + "id": "call_1654638__thought__" + GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + "type": "function", + }, + ], + "images": [], + "reasoning_content": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + }, + ], + "provider_specific_fields": { + "thought_signatures": [ + GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + ], + }, + }, + "provider_specific_fields": { + "native_finish_reason": "STOP", + }, + }, + ], + "usage": { + "completion_tokens": 210, + "prompt_tokens": 358, + "total_tokens": 568, + "completion_tokens_details": { + "reasoning_tokens": 150, + "text_tokens": 60, + }, + "prompt_tokens_details": { + "text_tokens": 358, + }, + }, + "vertex_ai_citation_metadata": [], + "vertex_ai_grounding_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_url_context_metadata": [], + }, +) + +GEMINI_4_ARGON_THOUGHT: Final = ( + "**My Analysis of Quantity Validation**\n\n" + "Okay, so the user wants me to start by looking at `shop/validation.py`. Right, that makes perfec" + "t sense given the instructions and the task. I need to understand how the system handles quantit" + "ies, especially zero, negative, and potentially non-integer values. It's crucial for me to have " + "that foundational knowledge before I even *think* about suggesting a fix, and the system wants m" + "e to ensure this is understood and considered.\n\n" + "Therefore, I'll state explicitly that I need to examine the `validate_quantity` function within " + "`shop/validation.py`. This will allow me to properly evaluate how quantities are being handled i" + "n detail.\n\nNow, I will use the `read_file` tool on `shop/validation.py`.\n\n\n" +) + +GEMINI_4_ARGON_THOUGHT_SIGNATURE: Final = ( + "Er0FCroFAWkUfRNrM27jPbX+vFoKJwoTVY0Na19E0vke/4MAfKRQYmXkW61lN+IKyH0lUQrByoh3xOQWzUqZrNCCJXa492UF" + "RkKVWhi+PVV+fwUBYsmlj7U9Yn379+WUNYt8rkl04KN7jmZJqzrP7T0A58rciLtp4t8vw1I+yi0+OZRBrvg9iU0gyCeWldbp" + "icQEAiwJBKYIJ9iiy5nFwhwbjANtiqo3WflAuBar1rWfnOq7aU3OOu6x5ZWKVABmTylKPBVCZeVn5ey5t/XJWxPPCF6ObIiF" + "V99T6S3OI+5wcC1HmHcl3LTCUHsSleVynLZ6Xpg1kpPmCEsFneXBEDqTgTjtYVSyuO/3dqnIM6Z/0OiPp+yoh1cjQguM72xB" + "2cqE+/OISSZTn85bhcXKXAQYHD4K0rOC7SL2bJVFtMOIrwby5CL4bFWiwFV8TLQzgHzE0Yhs2/tfGezXrofVaI0HApsUEKDz" + "F4zitTKD5pzw6k0JHOlZ9iETqCnPcQwhw5H2w+QYqn6M7D7HUQvYUF5ZhVWToq55xi/x261Q0K/a3LHUZuBZrYHpYP8NoV29" + "3v9Vibad/rPZK80wLDSgl+U/HAPuy/nBSoYM4IzQun5AkN6XTCHEM6cKUHLY6GzFTMWiWhBH9EBAUTP34OF4ztaNLxuZvjUn" + "X/iSiKRZj4cFHN74cfDJBj/SdRo/hlWVXl7D3TXAF9UOsjj3Hq5vjoSyIMJjLAoEU0ZxfrqkHGXE0kKHLqiuZaC9lzK0UvrV" + "V0MoQ+vf2NkzuN92hjMoFL9yi6DZgogR3B8XCUpghB4Jjx4CPX212Q4kPBxPYAuq7c2+VAm3yX3IDVmDnzENBNBKlqmXkjf6" + "LMdS+HZdhDwno+M5sq17u75h+FPfOQM8JXh0T5B8zAuQM6jtaKwqANJJYK9/wJM7zBQ5W1PnvaE=" +) + +GEMINI_4_ARGON_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_4_ARGON_THOUGHT, + "thought": True, + }, + { + "text": ( + "I will read `shop/validation.py` next to see how `validate_quantity` handles zero, negative, and" + " non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "start_line": 1, + "end_line": 50, + }, + "id": "call_1426689", + }, + "thoughtSignature": GEMINI_4_ARGON_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 414, + "candidatesTokenCount": 69, + "totalTokenCount": 628, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 414, + }, + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 69, + }, + ], + "thoughtsTokenCount": 145, + }, + "modelVersion": "gemini-4-argon", + "responseId": "4CbQarGpL8SGz7IPxf-zyQ0", +} + +GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "gemini/gemini-4-argon"}, + expected_provider_endpoint="/models/gemini-4-argon:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request=GEMINI_GENERATE_CONTENT_REQUEST, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_4_ARGON_RESPONSE_FIELDS}, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "gemini/gemini-4-argon", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": ( + "I will read `shop/validation.py` next to see how `validate_quantity` handles zero, negative, and" + " non-integer quantities." + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": GEMINI_4_ARGON_THOUGHT_SIGNATURE, + }, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "name": "read_file", + }, + "id": "call_1426689__thought__" + GEMINI_4_ARGON_THOUGHT_SIGNATURE, + "type": "function", + }, + ], + "images": [], + "reasoning_content": GEMINI_4_ARGON_THOUGHT, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": GEMINI_4_ARGON_THOUGHT, + }, + ], + "provider_specific_fields": { + "thought_signatures": [ + GEMINI_4_ARGON_THOUGHT_SIGNATURE, + ], + }, + }, + "provider_specific_fields": { + "native_finish_reason": "STOP", + }, + }, + ], + "usage": { + "completion_tokens": 214, + "prompt_tokens": 414, + "total_tokens": 628, + "completion_tokens_details": { + "reasoning_tokens": 145, + "text_tokens": 69, + }, + "prompt_tokens_details": { + "text_tokens": 414, + }, + }, + "vertex_ai_citation_metadata": [], + "vertex_ai_grounding_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_url_context_metadata": [], + }, +) GEMINI_3_5_FLASH_THOUGHT_SIGNATURE: Final = "ErkECrYEAWkUfRORlQ5j/gkLmyx08FVeIMgg57FJXrjcLwBlt9iuELI6Uc5h+NR/Vm+gEsbdZ5mgFTmKjPF9K/eft8IlXMKm6odJjQOgOYsbt2JHzANo0bpfTmlA6fIi0G2zLbvBVASA6Bdxu1aPQuO4voioQRwm2vomRxH1YbWp8sKXk0DBVbefosldrL0zLJFsi5dFYCtPvw0n9olPVgptHzEdiyXqG+63aPxTooARRQutUH0XWAKR0V+P7qWPt55QKlLaKQqKeBndg9JrEplJihg1sp++y+NAi18fsqXteUS2zIeDtdePGM/GS5oVibwE25zJoziPRdtJhGasFSaA7a3znhW9PF0pBIPAIRPKE4NsQ0FRhpy7ksIXY+0uJ4N+WPPejrtKK6z5x+P0tFkFP0ZNNPM8FZbir1ncVhVxkZS/wWmhc/8TZoRA9ghlTpYhHJ+C4fRVqQqnyRR3SDpVTzB4/sCjBlb434dTH0U3jB4h6V9b/Zx4k4pwUwZTNr2FfgOt2bR7u05DOa+H73OzsNG6zBnMYgBndQdRgk58+l4+UcZdpGKB0lkbHdfD2bminBypEmeJKNRpuc7Smuu0YxcZiY03tzzhHdrUmItqC39OEr2CzRcT9DjFpiWydo3ej9ZkEXeyxoMCckpMmGWh6xbjAnX9gkrPFmUE2rJqblDJWa51i6u/p9Y6ciCq6j4lAy9eBfULIRGQt9pKOXagltOnX0vR0MTDYgWe4dVDSzCI1TV7X/SQYjo=" @@ -78,155 +467,3 @@ GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( "vertex_ai_citation_metadata": [], }, ) - -GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = "EtsDCtgDAWkUfRODYoLRWC+DajYQxOvsLgPh0m8j4NTnd7BflzgPBKFfPW+PU1XsMQuEzviI1qk5mYI0qCfOQNf84PAXXvFA5hMYl+YObaND4G+ZtCdYcolFVfPJQqgK6Kpv20n9hZfLt5JzOS2+HRCLZaokIsZFadN++wqEeEkWQhnKdLGH1lM0fn8Fj/pYq95YLGnB90B8Oaj4qyG6ost2dzRAeAzFSXAko1mD/IgsDrDhEumngCqotdAbPW4jUGYOGDpoXLrBzQZvGa9blRC3ep6NLT0EYMnXImLFoZaLLIBMzVDsmmL0qOg4Gu+uNJlY6cDmtqRgkrcvuvGhh8+lrjUMJVigSsTAoKsTnT3OyCqdNqa+R2aD4WTl1uBFyGY7yXpZ9skQPkV210QNOylZ6exaMA51+W/mohL5j5+OJX7xtVfRIpjp4e0PLkEPxnuviX6OU4ykWZSSiztSXzogbrmnwP7faclRXTXHE5pFlM8y7gDJ9NEwxd5vvIMLdaK2dRWBNtvjI3Cj56sLKygH/j0mQSnt3PonE6Jv6XMJDPbuIzsjzX8t/j3u6WP0bVGnifNNEfIqt6YcKgeWlUTnabRGkWg/fBSyYbiTTEIGhketv57hA9oH7jJZbw==" - -GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "gemini/gemini-3.8-flash", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, - expected_provider_request={ - "contents": [{"parts": [{"text": "Say hello."}], "role": "user"}], - "generationConfig": {"max_output_tokens": 1024}, - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], - "role": "model", - }, - "finishReason": "STOP", - "index": 0, - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 2, - "totalTokenCount": 120, - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10}], - "thoughtsTokenCount": 108, - "serviceTier": "standard", - }, - "modelVersion": "gemini-3.8-flash", - "responseId": "5BjEaunaCtadz7IP-8WKsAk", - }, - expected_litellm_response={ - "id": "5BjEaunaCtadz7IP-8WKsAk", - "created": ANY, - "model": "gemini/gemini-3.8-flash", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_8_FLASH_THOUGHT_SIGNATURE]}, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - } - ], - "usage": { - "completion_tokens": 110, - "prompt_tokens": 10, - "total_tokens": 120, - "completion_tokens_details": {"reasoning_tokens": 108, "text_tokens": 2}, - "prompt_tokens_details": {"text_tokens": 10}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, -) - -GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = "Ev8DCvwDAWkUfRMrwgm49bezSdfm90OeW6KeR4kKqExF8s+EorBlW6NWc6XLqdQ2dA5xZ4p0CPNrjuxemR5dt9ch3qMTSgIHTcKKAnxJ0eqsjR61EJWFP4JNdigiymymc7UNs/zLZer+qPH2XQLD9r85O3NVBeupYk6xy6395CZYygF9oVJD3WNXwlefvThnvH/3rDsnO0FBfcrvxRHiSVTD1Moe+uTVV2w3vKKSCxUb64w5lquEjFx+AO/jiJIc3McPvAOvUr0I/2fMCWLcO7Y5sV6zuN8qpaQKitC/Ev09cl3SAbKzwjO3gBgYVmF5PcPY5HT8S3bwCez2aOgX7BCN+FItlZ4wMsZStLIY38XMQtbibVRHmiufN86IMkoD8Yxb1lnK7aaS+anSPkn4M4zkkiwAUdjt14k6JvMBk9J4duvTHLP0BiSaLLkTe6Ufj9cRMxZKCP0ew4DiLMwejuQoSpC4aLP1gmli4eOUcyq/g2/o0kByY8Fl2vb54eiXADp4fhhIEAZFe4J/0x29ZZVKx8KEInDahP7tslzwah3PWfX/K1jXzqX93mo4a/0Ec1bM/sqwiEWSHNhrPpyXcTkmWQ1Bot+PTARnzAjeXvoTR5sajbNKj1SdCC0RY7zb8OqGZHDQxgNw7ghTjYMTdBJsdCMQfnl3dvqmw6zM/aplZg==" - -GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "gemini/gemini-3.1-pro-preview", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", - expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, - expected_provider_request={ - "contents": [{"parts": [{"text": "Say hello."}], "role": "user"}], - "generationConfig": {"max_output_tokens": 1024}, - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], - "role": "model", - }, - "finishReason": "STOP", - "index": 0, - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 2, - "totalTokenCount": 115, - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10}], - "thoughtsTokenCount": 103, - "serviceTier": "standard", - }, - "modelVersion": "gemini-3.1-pro-preview", - "responseId": "5RjEaqfwC7XYqtsPp6TuoAY", - }, - expected_litellm_response={ - "id": "5RjEaqfwC7XYqtsPp6TuoAY", - "created": ANY, - "model": "gemini/gemini-3.1-pro-preview", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE]}, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - } - ], - "usage": { - "completion_tokens": 105, - "prompt_tokens": 10, - "total_tokens": 115, - "completion_tokens_details": {"reasoning_tokens": 103, "text_tokens": 2}, - "prompt_tokens_details": {"text_tokens": 10}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, -) diff --git a/tests/integration/translation/chat_completions/bases/openai.py b/tests/integration/translation/chat_completions/bases/openai.py index 4c33f5f02e4..83d2bef9dd2 100644 --- a/tests/integration/translation/chat_completions/bases/openai.py +++ b/tests/integration/translation/chat_completions/bases/openai.py @@ -1,7 +1,11 @@ from typing import Final from integration.translation.case import TranslationTestCase - +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.openai import ( + OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, + OPENAI_RESPONSES_RESPONSE, +) GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -276,280 +280,632 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) - GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "openai/gpt-6-luna", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "openai/gpt-6-luna"}, + expected_provider_endpoint="/responses", expected_provider_headers={ "content-type": "application/json", "authorization": "Bearer synthetic-openai-key", }, - expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "model": "gpt-6-luna", - "max_completion_tokens": 64, - }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, "model": "gpt-6-luna"}, mock_provider_response={ - "id": "chatcmpl-EVhxzlhJ6vfxv0dDB3QzTZvHJWTno", - "object": "chat.completion", - "created": 1791225883, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6-luna", - "choices": [ + "id": "resp_0a2459714d575226006ad02398406087d0a2501571a0097cce", + "created_at": 1792025496, + "access_programs": {"cyber": "daybreak_blue"}, + "completed_at": 1792025499, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect." + ), + "output": [ { - "index": 0, - "message": {"role": "assistant", "content": "Hello!", "refusal": None, "annotations": []}, - "finish_reason": "stop", - } + "id": "rs_0a2459714d575226006ad0239a42d087d08ab86bc19873f8f8", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0CObTtOHdIl0AomNdpnN-jGzO6wbopH5o7XAO-1__huttxkYQhzVBAV_TQ_Y-LHedrilDsn2OYqDrSZke4WPG2uMFL0z1kvWcj1zShAuI3bzLB4637l7tqQeM41ncLLPD2gfy4gS1G4oX-vWl-cJK5apCct8b6VXdWClY9H7OmLyrtYeyeTcvacBq3cJf6OJg6zI2f4O-rPsifmcGdXwojO8JWhliGBJX_98gjGwKEdBAVJHFXP57xEZbCd9b2ISUofeSbdVfvzCnwsRs5Bx40pQf_abIM99JdNNictIr0Zim2jiIEh5hfOJhlu6vRmaLT3eJ9fRE_ao_wkSaCllTwzH7z-QUBSSXmj8SqQCp0m-WExL4SPKrTLhQteqTRDx4F833tJNJbdAe65PyO_2cChbvqmIU9BMB0dftkbfWdt3JwP8be7_aSG-c250rTCB373oI8Xf5vBQcqVhJ3hfu-1ddFjWgFdk7b7V1nlBKsKK3_EzCNCMmcoGwKFShIZRL9RxyBYiStLjUT3PA9DtBay6wKyW62mG2JJPQxknrOE0BBmcD3Kg4EbH9jgThKxv4zQkFTSl_-gsoSFFnUbhRDCcSQpirefKYf_xBWpBU55_jjHWfkhZXfOuU4eArnD6g4Lt7T3wd9SQyRh3I5xdtzNUxd4fswu99REgL_S3tfUmPohLrFGBywb4SSFbqaz7HwR8y3NN9Aet7XVdTskdfv0g3_sBKQyCf1o3-hc4lNBFxXFaEEFXMQTPZKOjfPJNP7ZKxfueiwadGg0aVV-dPCAMnMuSncc5RC17f9ohZyGncTQHT1lLJJPOiAcQPThZABVu4Ptnuhn2oXs0HGk_q6y3cWDXOZOojrFncvDGHC8dUx2KNhNKsDGVGteHay8UDQBmEnSMez2mfHdCRRcuO6R9ZBWV-7TMAMxYgJ_nnGk-n2WAEmIrnWtSQkvInKtYf3A8bB9nyJ6Beu8IL20baT1SD3mtnm4y-ozQ8uuVsuVRNAatotjX23kjzx0kXpiS9xNeD5nYmxIDpO9TAb3nx2F5KPbEYbOB-7B_yvjteQFk6EfApTmwH9xX2nOGfN42ttX063EtBK4IvSbbGWLbjmfbgHB_hdn6Iev7zJxZ53M_WyPAMWcbvx7OZijvfAznAwGyRDDFla-CZmQhiFDPq5GwVsLwSHK-XIi-nPMJfBaBBXdP03KqG6whBNA2q6ZvyAvdr5hyFab6atRocfK6eSGbhpmrY5VTDEW7P1jqmG2myIFEMtgNCw7HA9DC11RD0cRk6e2Fwl_xrmHm9G7NL06RQT5NaGJIBRRHRt9fqAj4JlWFbXzj866Ob21ApfBHvP7GLrTWQeNJf5IcJYrZbgTK0JOn0twLTf0ekZoDzrw7qu1a9OnC3rsDQsMYRNtaOkAyHOfQhcQwZmg264AIhqmy_69fUZxBuLq-aLIVSDiJUujKRqiTtjAlvdqXObg-1lnDf9IhSB4qXn0Vcraows4g9b2-gc2t6GoGwqf3Cv7fXFK4jWajNsNenMgEjHyXVTGRY5fIBLBhoRzOcuy8FuIXN6WeYj6JAbnGfnB4Z4yssWJrSiqcH4X5YUHSZDYkB3w9dsZpTDCr8kNoW6oJrnrX6tegNQJUIi6iaQHR2vxptmnvueTKbC3sj2J0ICPWhLKbjOHMOHgGoLaT_s-SiH4CjrvoZALYOWMKyP0EMghJl9mNFLt-mnexO6Egm6J9xAuglMUxFm1txFuDBAp7jq1TR6z1mLm4-ZorpVIvbJv85FS4qQmpGpY_nU1IFTwcKET4skSsJk9utzfW5nfG6A9kejsG-eJNg9yatnenrglGNDs5RmfBKvbvr_yGK1agM3n47n9So8r7G46CviCHDoyx0nvQwog83YYAQOxEd8mlziu0ZmBrEwWxSMeGyYIPVGOWr3FrutMX7ySxxN9uk7oNzZFl9O4PoZwaCCipA0KDAVXiJ0A36Zp2sqnIeecXkWJEZHxynZ6Y8mOFn4SM-JrZ_PJBvWPpPEYTTKSCUrBh1ytBzXN2zU435TJ8I1hGBiG6atVCJ3DK4UtBhzYMwkG6Yj6jhXGBsNBUBaN8-vmCcda620JebM1U4dCtLhwsW_4J2B-S9gaYIXWtV10kIO-yKW_L8QCZQAtB44pOfKwPjy_n4PHoMV4IibbNDCDclVl8cAnO7-RJH5sewDmU1M_u5llf1x6wd8nWBKh1_kBlc6jPIZ8hkVTxy6QvthAYPV5_BPGtOEhbxj8po60tBU5d-kZMEkLBFQdKjiwTOMTBBe5SsdC32vnqL7ZZEj5ZK6xiCY4jYA8ykTyE9jTtx_rdnSpPMLviBw2gbOmlOVGoUw_AX71pojMTYDuVT_mnXrid5Dd7vVaBe8-3a9p0dnxX29k5JEo48yT9DY5JqXo2Dvg5YAadq4uueww8VGOrM25V", + "summary": [], + }, + { + "id": "msg_0a2459714d575226006ad0239af77c87d09bf21be68f806551", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to see how it handles zero, negative, " + "and non-integer quantities.", + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0a2459714d575226006ad0239b46bc87d0ad9e2b753765029b", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_zySVmQauaNPsuWHCle7BA4CX", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, ], "usage": { - "prompt_tokens": 19, - "completion_tokens": 5, - "total_tokens": 24, - "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "audio_tokens": 0}, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0, - }, + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 141, + "output_tokens_details": {"reasoning_tokens": 76}, + "total_tokens": 453, }, - "service_tier": "default", - "system_fingerprint": None, }, expected_litellm_response={ - "id": "chatcmpl-EVhxzlhJ6vfxv0dDB3QzTZvHJWTno", - "created": 1791225883, + **CHAT_COMPLETIONS_RESPONSE, "model": "openai/gpt-6-luna", - "object": "chat.completion", + "id": "resp_0a2459714d575226006ad02398406087d0a2501571a0097cce", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello!", + "content": "I’ll read `shop/validation.py` first to see how it handles zero, " + "negative, and non-integer quantities.", "role": "assistant", - "refusal": None, - "annotations": [], + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_zySVmQauaNPsuWHCle7BA4CX", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0a2459714d575226006ad0239a42d087d08ab86bc19873f8f8", + "encrypted_content": "gAAAAABq0CObTtOHdIl0AomNdpnN-jGzO6wbopH5o7XAO-1__huttxkYQhzVBAV_TQ_Y-LHedrilDsn2OYqDrSZke4WPG2uMFL0z1kvWcj1zShAuI3bzLB4637l7tqQeM41ncLLPD2gfy4gS1G4oX-vWl-cJK5apCct8b6VXdWClY9H7OmLyrtYeyeTcvacBq3cJf6OJg6zI2f4O-rPsifmcGdXwojO8JWhliGBJX_98gjGwKEdBAVJHFXP57xEZbCd9b2ISUofeSbdVfvzCnwsRs5Bx40pQf_abIM99JdNNictIr0Zim2jiIEh5hfOJhlu6vRmaLT3eJ9fRE_ao_wkSaCllTwzH7z-QUBSSXmj8SqQCp0m-WExL4SPKrTLhQteqTRDx4F833tJNJbdAe65PyO_2cChbvqmIU9BMB0dftkbfWdt3JwP8be7_aSG-c250rTCB373oI8Xf5vBQcqVhJ3hfu-1ddFjWgFdk7b7V1nlBKsKK3_EzCNCMmcoGwKFShIZRL9RxyBYiStLjUT3PA9DtBay6wKyW62mG2JJPQxknrOE0BBmcD3Kg4EbH9jgThKxv4zQkFTSl_-gsoSFFnUbhRDCcSQpirefKYf_xBWpBU55_jjHWfkhZXfOuU4eArnD6g4Lt7T3wd9SQyRh3I5xdtzNUxd4fswu99REgL_S3tfUmPohLrFGBywb4SSFbqaz7HwR8y3NN9Aet7XVdTskdfv0g3_sBKQyCf1o3-hc4lNBFxXFaEEFXMQTPZKOjfPJNP7ZKxfueiwadGg0aVV-dPCAMnMuSncc5RC17f9ohZyGncTQHT1lLJJPOiAcQPThZABVu4Ptnuhn2oXs0HGk_q6y3cWDXOZOojrFncvDGHC8dUx2KNhNKsDGVGteHay8UDQBmEnSMez2mfHdCRRcuO6R9ZBWV-7TMAMxYgJ_nnGk-n2WAEmIrnWtSQkvInKtYf3A8bB9nyJ6Beu8IL20baT1SD3mtnm4y-ozQ8uuVsuVRNAatotjX23kjzx0kXpiS9xNeD5nYmxIDpO9TAb3nx2F5KPbEYbOB-7B_yvjteQFk6EfApTmwH9xX2nOGfN42ttX063EtBK4IvSbbGWLbjmfbgHB_hdn6Iev7zJxZ53M_WyPAMWcbvx7OZijvfAznAwGyRDDFla-CZmQhiFDPq5GwVsLwSHK-XIi-nPMJfBaBBXdP03KqG6whBNA2q6ZvyAvdr5hyFab6atRocfK6eSGbhpmrY5VTDEW7P1jqmG2myIFEMtgNCw7HA9DC11RD0cRk6e2Fwl_xrmHm9G7NL06RQT5NaGJIBRRHRt9fqAj4JlWFbXzj866Ob21ApfBHvP7GLrTWQeNJf5IcJYrZbgTK0JOn0twLTf0ekZoDzrw7qu1a9OnC3rsDQsMYRNtaOkAyHOfQhcQwZmg264AIhqmy_69fUZxBuLq-aLIVSDiJUujKRqiTtjAlvdqXObg-1lnDf9IhSB4qXn0Vcraows4g9b2-gc2t6GoGwqf3Cv7fXFK4jWajNsNenMgEjHyXVTGRY5fIBLBhoRzOcuy8FuIXN6WeYj6JAbnGfnB4Z4yssWJrSiqcH4X5YUHSZDYkB3w9dsZpTDCr8kNoW6oJrnrX6tegNQJUIi6iaQHR2vxptmnvueTKbC3sj2J0ICPWhLKbjOHMOHgGoLaT_s-SiH4CjrvoZALYOWMKyP0EMghJl9mNFLt-mnexO6Egm6J9xAuglMUxFm1txFuDBAp7jq1TR6z1mLm4-ZorpVIvbJv85FS4qQmpGpY_nU1IFTwcKET4skSsJk9utzfW5nfG6A9kejsG-eJNg9yatnenrglGNDs5RmfBKvbvr_yGK1agM3n47n9So8r7G46CviCHDoyx0nvQwog83YYAQOxEd8mlziu0ZmBrEwWxSMeGyYIPVGOWr3FrutMX7ySxxN9uk7oNzZFl9O4PoZwaCCipA0KDAVXiJ0A36Zp2sqnIeecXkWJEZHxynZ6Y8mOFn4SM-JrZ_PJBvWPpPEYTTKSCUrBh1ytBzXN2zU435TJ8I1hGBiG6atVCJ3DK4UtBhzYMwkG6Yj6jhXGBsNBUBaN8-vmCcda620JebM1U4dCtLhwsW_4J2B-S9gaYIXWtV10kIO-yKW_L8QCZQAtB44pOfKwPjy_n4PHoMV4IibbNDCDclVl8cAnO7-RJH5sewDmU1M_u5llf1x6wd8nWBKh1_kBlc6jPIZ8hkVTxy6QvthAYPV5_BPGtOEhbxj8po60tBU5d-kZMEkLBFQdKjiwTOMTBBe5SsdC32vnqL7ZZEj5ZK6xiCY4jYA8ykTyE9jTtx_rdnSpPMLviBw2gbOmlOVGoUw_AX71pojMTYDuVT_mnXrid5Dd7vVaBe8-3a9p0dnxX29k5JEo48yT9DY5JqXo2Dvg5YAadq4uueww8VGOrM25V", + "summary": [], + } + ], }, - "provider_specific_fields": {}, } ], "usage": { - "completion_tokens": 5, - "prompt_tokens": 19, - "total_tokens": 24, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - }, + "completion_tokens": 141, + "prompt_tokens": 312, + "total_tokens": 453, + "completion_tokens_details": {"reasoning_tokens": 76}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, + "access_programs": {"cyber": "daybreak_blue"}, + "billing": {"payer": "developer"}, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, "service_tier": "default", - "system_fingerprint": None, + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, }, ) - GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "openai/gpt-6.1-sol", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "openai/gpt-6.1-sol"}, + expected_provider_endpoint="/responses", expected_provider_headers={ "content-type": "application/json", "authorization": "Bearer synthetic-openai-key", }, - expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "model": "gpt-6.1-sol", - "max_completion_tokens": 64, - }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, "model": "gpt-6.1-sol"}, mock_provider_response={ - "id": "chatcmpl-EVhy2Sc0BN5sjJZK2Wa6l3CtsQ2ot", - "object": "chat.completion", - "created": 1791225886, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6.1-sol", - "choices": [ + "id": "resp_0e8134ce088f63e7006ad023a2e5b887d0a0f422419e0bfbe7", + "created_at": 1792025506, + "completed_at": 1792025510, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect." + ), + "output": [ { - "index": 0, - "message": {"role": "assistant", "content": "Hello!", "refusal": None, "annotations": []}, - "finish_reason": "stop", - } + "id": "rs_0e8134ce088f63e7006ad023a3ff5887d0801528ceb0190f4b", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COmpSSf5bN4H0xC2pbhfp_niGPAm58n4-JyZsHy0ytyhg6KzzTZiR0cjuNAHvZR_w8oavaY1zzH-MkWItVhAx-PQJQ560Da0Dwqi_wwJ0831R8W6lxyeFxgBZXpZ4_TsbT1qwt7IwVC6fJE4gDeFGiuo4H7plTfbE7C2DaO_0fkI5Zt_iRZLA3VOJPFnzegipK4A5AxckZNVZLI705xzMfi7KU6J886jCmsqCxjrQKLwP4fB0jQoRidBCensrctV2C7imXgTAlTwOysSd8gf7Yhu7A9qD1rUj_jvLGJCiqBCe9Cyv9I7cX82VewV4029mGOngvQOUAgVbqT_i9LkQ3HfBXu4SYTK_LXU2P0lw_P69qKYGTZfdwxqui8VVtgJI7kDrZY-l7DvqaIncKJ-LgDJBvxaYDMM7H1Co68Yd64RM3U1v4kTezKN_MBg-1Ku0m1e3Zi7YZDHOHkFDDGFXzXCpbGkgDJCHsG7i-UTE1uAjkgdP9h0MUFosMYB8pl2NxtN8NnEHsBfyYBV56Sms8ObilQaK2eEWvnDH_wkGLt2RAGOJwfovSEwKnWgOemEZtA_IYbxhRNupRJkPy0oJ4xPdAxIICncSkwaF_NMYeIetF-0tvqK2mqWZVnI13w_CPUW0WI0nlfSnlqO-7gD4Xh80-cvFAqr44X_yaRODPOOc6aWoXb_23A2AoFW6vEKufFf3zni2JEHiCREgvUAp8p931WYo7H_lDpOqIHGiQVjvRfUN9pRPnpPbF6NwCUNalgQPt4UZal_ubh3kdF1NRqKuTQMcalPfuw7Zdc7-ZFTV3JdUApBwsF2jGFXzAfmwHytnOohHOAm79n_7ORaJV3ecKjyg0t-N777eu-7LJX79tvy1tNXR-0DsodobU-mHuyxqC4Qhmqk8jC1IeNG7du1IUwpFJnOyM4O1D6xUYGslBWXsIa_exQpTkxH0PYSYeCGA9GCeWsrn8gSkOgwBmtsC56Bow8VldAr_t_AS5UIrSUsNvn1SF-nZJKbe1mvL-egWOuQq8dZb3wEF7vVxqgn4_f54pGXyDeEdxPdxAKhqT672MKb3l_K11Yn10kbhNr1XqX-m323zLcNizjPlH8rJGUWht7RGQd6VOxEjmhKC_ZWFsu7wQgqsGX0BQ2e-6hZE2Kq_9u_MfSieYJyt9jb6XXrqefSP2tyWsrrS6HrBBrSkh2d5PfiP4MQ2gTesP9tj-yVlq4JU3FZMgqS5W7m2BmhD8gGonaLSK5zJboTYjpZHoS0swfxX2_BJDiARNrPFZ58t6sSXDZauTZq8CbdLdXUBA6RaO1kp-yscPIuB2DHjPjPUVrRYbNWSPIcjqHDYY11UHPs7lWL8U5j-j3EfYwiwQ85YnVfoB7jzJFeiywAUvbum20MJXBOE2wrvrR6zXJAGYlEwxKbl_ibZULRRuJ-VSGeijiDNtavD5v5-AAmow5yPuuVUH9_kx7Un8ZbbESZH1rptCToDmxeRrIa51Xw8fIbONz1fVNzljCUvqvsoyjPQAlaZy_YfLIOYD5MVhVX8XiR8mIeSna4ELdWXu8EEmPWyl676GbSZvMzQmBVqDFeELEusHm_UNX3oI5oOMJQ8ZA9p-JyJlx8LWDe3wqTaKKzoajxn2Wz6qvjKFStkGrk0voFzEPEnBbIAIPkh7Uv2ORupzlQeF6Hor11DxlDAIaUQf1344U37C-a1ODbqL8tg7HOm1WSKdwWeKG2-SYk9K8yB1hEZuqmxHquiXybUU-EYrQFGv88d-K0ou-wAjolMdXlIUaT5-OmqA8HCprI4Nr1BdxQYLLwlNyxm5Y610a1Bd9clo8YbW-I_zSNVm74nF1_HzKP3Ym9H3_f1ZWj55BOAw9i1jmk6kb7P2SAomQ9WVCHvhbyPUhrMMlDxc5VpQOMlFRIZjCEb4wirnD4oonK5mt_TY1l_XMc7DAFb91wpdF3dB6Nh8W2_YYC8_vHfvvSq5WtVKBVIBzZM7l188EX3zHUoe4a9pbQj2Yj2-fAF4qnae4DZ1IwYvoUrln3YETLC37Qb7c3cwM_-i9YLiB_F-qA-pVnmwH8Ah2a3cOF-sl50BE7ey_rHw0fhnflc2bicoLMA82gXdE3iuAsPX-sJvouXgLB1KYmFZt_8Hm3QlrT2c8mj_Jt10YdPC_haBCvzOQDvi5kfPM2upPrvgm7bPEgyjLZECEaQj7bjgAsZc7PqnNp0u41Quq2WxUbjMrE6BPjcrhF5Llx6PQjrz2d5_023M1xBMGxKNXaLNmyi91YpvC0Sqd9qUKndUQ6fPA4z03_FP7-UnzYPuJQaBX662f1Vd-sSUvvM4zdPw6YX-muGqHF5qld2He6eMaUUXhjDuwvy2LW5WRQfSbWJ89XsUYqDwDM0j8TEexma6UZ07uNV_xjse6P8JfL-L8BDiHkR7ZNRMDSYqsHQjqwoYZNUs25XwWPr7kjn-43H0mW_dXPcManZ6qBlKKETmdtr8CZrtNxZ1_TnHUeNkATzhmtzyLQRmouxUq6tbPMumlcXcwhvDBSKSR33a4LZG1kknf_YlTDLA4x7sHImDRMYgM9d6SYxBuTSq1OfTtSv7i4Wz5_Pef3iNFI8JH6Q6b24vED1lnWvVEImDKReDYdDLoNJr2BbSt6iZ0hsGSQ5EMsE3jlizNIBwdqI5z2MKTCYWAXSbxXe62zgaklRD6IoCER9h4SqjsnOh9esKu1bg-AkSa04Le9PP4i2whspq6Xa7f4DJpRYb-hEdY", + "summary": [], + }, + { + "id": "msg_0e8134ce088f63e7006ad023a5aa0c87d0aee7de2e31f58a5c", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to see how it handles zero, negative " + "values, and non-integer quantities.", + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0e8134ce088f63e7006ad023a615bc87d0b7aaafc731d1f086", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_sPhtuzlfkfNe16kBteKPS72V", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, ], "usage": { - "prompt_tokens": 19, - "completion_tokens": 5, - "total_tokens": 24, - "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "audio_tokens": 0}, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0, - }, + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 166, + "output_tokens_details": {"reasoning_tokens": 100}, + "total_tokens": 478, }, - "service_tier": "default", - "system_fingerprint": None, }, expected_litellm_response={ - "id": "chatcmpl-EVhy2Sc0BN5sjJZK2Wa6l3CtsQ2ot", - "created": 1791225886, + **CHAT_COMPLETIONS_RESPONSE, "model": "openai/gpt-6.1-sol", - "object": "chat.completion", + "id": "resp_0e8134ce088f63e7006ad023a2e5b887d0a0f422419e0bfbe7", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello!", + "content": "I’ll read `shop/validation.py` first to see how it handles zero, " + "negative values, and non-integer quantities.", "role": "assistant", - "refusal": None, - "annotations": [], + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_sPhtuzlfkfNe16kBteKPS72V", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0e8134ce088f63e7006ad023a3ff5887d0801528ceb0190f4b", + "encrypted_content": "gAAAAABq0COmpSSf5bN4H0xC2pbhfp_niGPAm58n4-JyZsHy0ytyhg6KzzTZiR0cjuNAHvZR_w8oavaY1zzH-MkWItVhAx-PQJQ560Da0Dwqi_wwJ0831R8W6lxyeFxgBZXpZ4_TsbT1qwt7IwVC6fJE4gDeFGiuo4H7plTfbE7C2DaO_0fkI5Zt_iRZLA3VOJPFnzegipK4A5AxckZNVZLI705xzMfi7KU6J886jCmsqCxjrQKLwP4fB0jQoRidBCensrctV2C7imXgTAlTwOysSd8gf7Yhu7A9qD1rUj_jvLGJCiqBCe9Cyv9I7cX82VewV4029mGOngvQOUAgVbqT_i9LkQ3HfBXu4SYTK_LXU2P0lw_P69qKYGTZfdwxqui8VVtgJI7kDrZY-l7DvqaIncKJ-LgDJBvxaYDMM7H1Co68Yd64RM3U1v4kTezKN_MBg-1Ku0m1e3Zi7YZDHOHkFDDGFXzXCpbGkgDJCHsG7i-UTE1uAjkgdP9h0MUFosMYB8pl2NxtN8NnEHsBfyYBV56Sms8ObilQaK2eEWvnDH_wkGLt2RAGOJwfovSEwKnWgOemEZtA_IYbxhRNupRJkPy0oJ4xPdAxIICncSkwaF_NMYeIetF-0tvqK2mqWZVnI13w_CPUW0WI0nlfSnlqO-7gD4Xh80-cvFAqr44X_yaRODPOOc6aWoXb_23A2AoFW6vEKufFf3zni2JEHiCREgvUAp8p931WYo7H_lDpOqIHGiQVjvRfUN9pRPnpPbF6NwCUNalgQPt4UZal_ubh3kdF1NRqKuTQMcalPfuw7Zdc7-ZFTV3JdUApBwsF2jGFXzAfmwHytnOohHOAm79n_7ORaJV3ecKjyg0t-N777eu-7LJX79tvy1tNXR-0DsodobU-mHuyxqC4Qhmqk8jC1IeNG7du1IUwpFJnOyM4O1D6xUYGslBWXsIa_exQpTkxH0PYSYeCGA9GCeWsrn8gSkOgwBmtsC56Bow8VldAr_t_AS5UIrSUsNvn1SF-nZJKbe1mvL-egWOuQq8dZb3wEF7vVxqgn4_f54pGXyDeEdxPdxAKhqT672MKb3l_K11Yn10kbhNr1XqX-m323zLcNizjPlH8rJGUWht7RGQd6VOxEjmhKC_ZWFsu7wQgqsGX0BQ2e-6hZE2Kq_9u_MfSieYJyt9jb6XXrqefSP2tyWsrrS6HrBBrSkh2d5PfiP4MQ2gTesP9tj-yVlq4JU3FZMgqS5W7m2BmhD8gGonaLSK5zJboTYjpZHoS0swfxX2_BJDiARNrPFZ58t6sSXDZauTZq8CbdLdXUBA6RaO1kp-yscPIuB2DHjPjPUVrRYbNWSPIcjqHDYY11UHPs7lWL8U5j-j3EfYwiwQ85YnVfoB7jzJFeiywAUvbum20MJXBOE2wrvrR6zXJAGYlEwxKbl_ibZULRRuJ-VSGeijiDNtavD5v5-AAmow5yPuuVUH9_kx7Un8ZbbESZH1rptCToDmxeRrIa51Xw8fIbONz1fVNzljCUvqvsoyjPQAlaZy_YfLIOYD5MVhVX8XiR8mIeSna4ELdWXu8EEmPWyl676GbSZvMzQmBVqDFeELEusHm_UNX3oI5oOMJQ8ZA9p-JyJlx8LWDe3wqTaKKzoajxn2Wz6qvjKFStkGrk0voFzEPEnBbIAIPkh7Uv2ORupzlQeF6Hor11DxlDAIaUQf1344U37C-a1ODbqL8tg7HOm1WSKdwWeKG2-SYk9K8yB1hEZuqmxHquiXybUU-EYrQFGv88d-K0ou-wAjolMdXlIUaT5-OmqA8HCprI4Nr1BdxQYLLwlNyxm5Y610a1Bd9clo8YbW-I_zSNVm74nF1_HzKP3Ym9H3_f1ZWj55BOAw9i1jmk6kb7P2SAomQ9WVCHvhbyPUhrMMlDxc5VpQOMlFRIZjCEb4wirnD4oonK5mt_TY1l_XMc7DAFb91wpdF3dB6Nh8W2_YYC8_vHfvvSq5WtVKBVIBzZM7l188EX3zHUoe4a9pbQj2Yj2-fAF4qnae4DZ1IwYvoUrln3YETLC37Qb7c3cwM_-i9YLiB_F-qA-pVnmwH8Ah2a3cOF-sl50BE7ey_rHw0fhnflc2bicoLMA82gXdE3iuAsPX-sJvouXgLB1KYmFZt_8Hm3QlrT2c8mj_Jt10YdPC_haBCvzOQDvi5kfPM2upPrvgm7bPEgyjLZECEaQj7bjgAsZc7PqnNp0u41Quq2WxUbjMrE6BPjcrhF5Llx6PQjrz2d5_023M1xBMGxKNXaLNmyi91YpvC0Sqd9qUKndUQ6fPA4z03_FP7-UnzYPuJQaBX662f1Vd-sSUvvM4zdPw6YX-muGqHF5qld2He6eMaUUXhjDuwvy2LW5WRQfSbWJ89XsUYqDwDM0j8TEexma6UZ07uNV_xjse6P8JfL-L8BDiHkR7ZNRMDSYqsHQjqwoYZNUs25XwWPr7kjn-43H0mW_dXPcManZ6qBlKKETmdtr8CZrtNxZ1_TnHUeNkATzhmtzyLQRmouxUq6tbPMumlcXcwhvDBSKSR33a4LZG1kknf_YlTDLA4x7sHImDRMYgM9d6SYxBuTSq1OfTtSv7i4Wz5_Pef3iNFI8JH6Q6b24vED1lnWvVEImDKReDYdDLoNJr2BbSt6iZ0hsGSQ5EMsE3jlizNIBwdqI5z2MKTCYWAXSbxXe62zgaklRD6IoCER9h4SqjsnOh9esKu1bg-AkSa04Le9PP4i2whspq6Xa7f4DJpRYb-hEdY", + "summary": [], + } + ], }, - "provider_specific_fields": {}, } ], "usage": { - "completion_tokens": 5, - "prompt_tokens": 19, - "total_tokens": 24, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - }, + "completion_tokens": 166, + "prompt_tokens": 312, + "total_tokens": 478, + "completion_tokens_details": {"reasoning_tokens": 100}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, + "billing": {"payer": "developer"}, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, "service_tier": "default", - "system_fingerprint": None, + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, }, ) GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "openai/gpt-6-astra", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "openai/gpt-6-astra"}, + expected_provider_endpoint="/responses", expected_provider_headers={ "content-type": "application/json", "authorization": "Bearer synthetic-openai-key", }, - expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "model": "gpt-6-astra", - "max_completion_tokens": 64, - }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, "model": "gpt-6-astra"}, mock_provider_response={ - "id": "chatcmpl-EZ37nFHKO9u4Sc7LwqNWTR0iHbIwn", - "object": "chat.completion", - "created": 1792022199, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6-astra", - "choices": [ + "id": "resp_0dd9c3f2f13006a2006ad0238bcfa087d0ae1a00e93e9dcfa5", + "created_at": 1792025483, + "completed_at": 1792025487, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect." + ), + "output": [ { - "index": 0, - "message": {"role": "assistant", "content": "Hello!", "refusal": None, "annotations": []}, - "finish_reason": "stop", - } + "id": "rs_0dd9c3f2f13006a2006ad0238ce43887d08314c8e2f0619171", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COPW81YCmA7VJWEaMeFZwZsixHXQg1hUZ8RoBpCteWdhDHmuIxYPWc-lwYTsH28yN7Hjlrgvtmvn5jtVILJozkIcXbLvyXzNi3hswFvpK0NOfkzIeepbxIkFPbEWmAHnGy12UpwFSaQHWNnq3n3_yx1QHg-6L84uhXb_gVLWbQpPfJSUHvqfymkwAH1GoqMKv5GRU1vQQzxs5O8WTfgNLyX0riq6mbYoO8jjRd-QzqkHf5w0fbNsQ0a1npvML0f6vzPdDsBKUrmDiwrd5HjM5tn62CI5ApllJFemVCTa_pXf0d7ws71b-g8d9jIVFtSoDe_oIZ6ZDTiGSRrdgRW3HzZG-hOS85nqwzYcUbXnZ_2dtZUelx1bFpGgH3udKhXIJ1YYXBoTsbYubHskPIWf0DE_rnbLUogslDawx9SBcYJ5dXTD0ipUYtIsW830XTVV1cOl2sFtp4SUrtJiV55G_r7sAxxUyQvF0-OlByS-wEbEvaQfylbFkHZUOcOs69DmgcGP1a_hdD5ZiiY3k1KG63JeIYFzDo1UomGzDFpbbihgq73-FD1JgHARe1vrwJ_0Y3BhhZjJ8UmeLGqyy4C6NQAmaLEg2iK4X-bzItSD-esAhENq-ZmUUKAY74SsGCdPK9srakfVz05rGN0_XKLkkoyW74UzWY89hkYAzrQOw5zGMghVKAioFcpyjX7JVz6fEdL4OWoaTknzhjnQKv1DEq2v8RsZVg3wWuU0t1WB562OQEIaScCVH4knjEtgCB8erZyStk65uwlXK5hQ6rof6Z4-jpd4xp89rAqNcX5OxnXG1SfST50s4C_YddfNe7Y0NQ9_yORz6TChuP3weUXnqlMOWHe4bMSTRLDITfLToWr4KwgajtQ_EmvCrbQWX5BHwSMHOCRKj92eSThzJ2EStpCmV-fihb6onbQk-x3i8pB0J3hkhfRzzsd2vEpy-E4RG9AgKhntN_gwP0Uez_hgEtfv9vipI8qVe-ivu168qU5N8siLollCok_DmIpnjsupxjNlifsv6LhKr97Y7S3TDnpdRL17Mt8lXJpuChunCDC7URHf990u4_j6tuQ9WLGKAq7F8lPoh8HzVdM5ozE-proWaEzFVd035l7q0dQxhQvw0JkhhcRMkjnmIFnTCQ3vAhHc2WCsy-gR5YI472RZoxU6x9q1KOVpTonw6ioUxZoB_QxZGxUT3FPdfMU6pOS0DZ0Ws1qL-49UOSuIleiEAhmX3E_lG_QD5dcxRbCOFHo3-JML3pTrXU49GZi7ZPqzmNcRSK7nxq1751e-YmBt_KYkqLI0sRM0Wwz1X1XTSv7S97iefaAj6xXhnlyc3hi5ofzj8iTGGF-mWYrC-F6GhCizpM91ePu6uugP-FWdtaQ5YXO-9Kq7mFwGB8NE-vftRZBr6p4TbGgXgrZk9rwMpgyC60WACV24UpJhaaBfnPgojE_nuskalMjAy41VA0CyMDCRBJ29MPE9R-dVMNIld9hfBSz2kMCaDXGw83fWPNAcUPQ2fvMBrBhkSLgTRhX7CDElb4foRJS6vTnTgW-es6mk9Ja-3-O013xo09WkqPyYieF1OIWyiALlUVqDtFYdWbUrIHUVV69D03XM3zIgy05HMI5uj46gT8l_X_nDnUI8zolKjB0STnTuzwwK4xvcwl7ank70dIvKatv62MrI-ZEp7QD00ZHeos5fh2X0YxNCsavuKF6UiT_hjWo-v9Sb2qJYKwiv_Bz_pqMqrQyQP-S35FkmWLzfu_ipbNxqE9z_u4WwvPUSdyAdMsJtBc41MJFtYAsBZc5jU0o7Sd2uiVQaUG51Q19jwq6MU68DxcTWukmdB4rQMd4EHYjtNJYr_zGGqzwJAO7gmKeAPxqbwbI5xg_JYGqdVfKBWKdZCCGljYW8DoimGJugt8OJvFFVu3wAOG9KRhaNtdmDnjYd24jxpqWtn5BZSNgwtQP1nh07a7aa-jsJmNpDxu9WH9fax7FXBjz8L9UlG47kBNTpQzKuW_8GRbe2Xvx5jJ0Rh9EFoW1XOiDap3s--MBUDwcRpQvxcU7Ip9gA7JNlsQJoglTvgZiQArlVlbkx7I6UOMHGzJ68UtuFyg3Z6eyAAXTBauozf2VCtbucac_f6kOC80xwZBxfTKG8iXlKOqa3_h_Q66kzPP6h6Ni4-FW-Z5b2aVCUGP_TnG4PkqS-ciRQz2CAdXcHkgy1bXbe-qWXpPBSYs0jnkBygX_v6sa3k8k2wXFenRz0nZ0D684585Axe5Kz5gvW5sR4n2LiN_x95cqDvXCLqzJ-T6kawaRBEoI2Ym3uw7tfOx3o1qyLAvZ9eX4c-dI1kVrC6WzJeeFy2OPOjY=", + "summary": [], + }, + { + "id": "msg_0dd9c3f2f13006a2006ad0238e66f487d0b8321be613a40623", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.", + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0dd9c3f2f13006a2006ad0238ee75087d0aeb5c9d34c955fb5", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_EEr3oYtEWt2tw0HIjEYCvhL4", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, ], "usage": { - "prompt_tokens": 19, - "completion_tokens": 5, - "total_tokens": 24, - "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "audio_tokens": 0}, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0, - }, + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 135, + "output_tokens_details": {"reasoning_tokens": 70}, + "total_tokens": 447, }, - "service_tier": "default", - "system_fingerprint": None, }, expected_litellm_response={ - "id": "chatcmpl-EZ37nFHKO9u4Sc7LwqNWTR0iHbIwn", - "created": 1792022199, + **CHAT_COMPLETIONS_RESPONSE, "model": "openai/gpt-6-astra", - "object": "chat.completion", + "id": "resp_0dd9c3f2f13006a2006ad0238bcfa087d0ae1a00e93e9dcfa5", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello!", + "content": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.", "role": "assistant", - "refusal": None, - "annotations": [], + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_EEr3oYtEWt2tw0HIjEYCvhL4", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0dd9c3f2f13006a2006ad0238ce43887d08314c8e2f0619171", + "encrypted_content": "gAAAAABq0COPW81YCmA7VJWEaMeFZwZsixHXQg1hUZ8RoBpCteWdhDHmuIxYPWc-lwYTsH28yN7Hjlrgvtmvn5jtVILJozkIcXbLvyXzNi3hswFvpK0NOfkzIeepbxIkFPbEWmAHnGy12UpwFSaQHWNnq3n3_yx1QHg-6L84uhXb_gVLWbQpPfJSUHvqfymkwAH1GoqMKv5GRU1vQQzxs5O8WTfgNLyX0riq6mbYoO8jjRd-QzqkHf5w0fbNsQ0a1npvML0f6vzPdDsBKUrmDiwrd5HjM5tn62CI5ApllJFemVCTa_pXf0d7ws71b-g8d9jIVFtSoDe_oIZ6ZDTiGSRrdgRW3HzZG-hOS85nqwzYcUbXnZ_2dtZUelx1bFpGgH3udKhXIJ1YYXBoTsbYubHskPIWf0DE_rnbLUogslDawx9SBcYJ5dXTD0ipUYtIsW830XTVV1cOl2sFtp4SUrtJiV55G_r7sAxxUyQvF0-OlByS-wEbEvaQfylbFkHZUOcOs69DmgcGP1a_hdD5ZiiY3k1KG63JeIYFzDo1UomGzDFpbbihgq73-FD1JgHARe1vrwJ_0Y3BhhZjJ8UmeLGqyy4C6NQAmaLEg2iK4X-bzItSD-esAhENq-ZmUUKAY74SsGCdPK9srakfVz05rGN0_XKLkkoyW74UzWY89hkYAzrQOw5zGMghVKAioFcpyjX7JVz6fEdL4OWoaTknzhjnQKv1DEq2v8RsZVg3wWuU0t1WB562OQEIaScCVH4knjEtgCB8erZyStk65uwlXK5hQ6rof6Z4-jpd4xp89rAqNcX5OxnXG1SfST50s4C_YddfNe7Y0NQ9_yORz6TChuP3weUXnqlMOWHe4bMSTRLDITfLToWr4KwgajtQ_EmvCrbQWX5BHwSMHOCRKj92eSThzJ2EStpCmV-fihb6onbQk-x3i8pB0J3hkhfRzzsd2vEpy-E4RG9AgKhntN_gwP0Uez_hgEtfv9vipI8qVe-ivu168qU5N8siLollCok_DmIpnjsupxjNlifsv6LhKr97Y7S3TDnpdRL17Mt8lXJpuChunCDC7URHf990u4_j6tuQ9WLGKAq7F8lPoh8HzVdM5ozE-proWaEzFVd035l7q0dQxhQvw0JkhhcRMkjnmIFnTCQ3vAhHc2WCsy-gR5YI472RZoxU6x9q1KOVpTonw6ioUxZoB_QxZGxUT3FPdfMU6pOS0DZ0Ws1qL-49UOSuIleiEAhmX3E_lG_QD5dcxRbCOFHo3-JML3pTrXU49GZi7ZPqzmNcRSK7nxq1751e-YmBt_KYkqLI0sRM0Wwz1X1XTSv7S97iefaAj6xXhnlyc3hi5ofzj8iTGGF-mWYrC-F6GhCizpM91ePu6uugP-FWdtaQ5YXO-9Kq7mFwGB8NE-vftRZBr6p4TbGgXgrZk9rwMpgyC60WACV24UpJhaaBfnPgojE_nuskalMjAy41VA0CyMDCRBJ29MPE9R-dVMNIld9hfBSz2kMCaDXGw83fWPNAcUPQ2fvMBrBhkSLgTRhX7CDElb4foRJS6vTnTgW-es6mk9Ja-3-O013xo09WkqPyYieF1OIWyiALlUVqDtFYdWbUrIHUVV69D03XM3zIgy05HMI5uj46gT8l_X_nDnUI8zolKjB0STnTuzwwK4xvcwl7ank70dIvKatv62MrI-ZEp7QD00ZHeos5fh2X0YxNCsavuKF6UiT_hjWo-v9Sb2qJYKwiv_Bz_pqMqrQyQP-S35FkmWLzfu_ipbNxqE9z_u4WwvPUSdyAdMsJtBc41MJFtYAsBZc5jU0o7Sd2uiVQaUG51Q19jwq6MU68DxcTWukmdB4rQMd4EHYjtNJYr_zGGqzwJAO7gmKeAPxqbwbI5xg_JYGqdVfKBWKdZCCGljYW8DoimGJugt8OJvFFVu3wAOG9KRhaNtdmDnjYd24jxpqWtn5BZSNgwtQP1nh07a7aa-jsJmNpDxu9WH9fax7FXBjz8L9UlG47kBNTpQzKuW_8GRbe2Xvx5jJ0Rh9EFoW1XOiDap3s--MBUDwcRpQvxcU7Ip9gA7JNlsQJoglTvgZiQArlVlbkx7I6UOMHGzJ68UtuFyg3Z6eyAAXTBauozf2VCtbucac_f6kOC80xwZBxfTKG8iXlKOqa3_h_Q66kzPP6h6Ni4-FW-Z5b2aVCUGP_TnG4PkqS-ciRQz2CAdXcHkgy1bXbe-qWXpPBSYs0jnkBygX_v6sa3k8k2wXFenRz0nZ0D684585Axe5Kz5gvW5sR4n2LiN_x95cqDvXCLqzJ-T6kawaRBEoI2Ym3uw7tfOx3o1qyLAvZ9eX4c-dI1kVrC6WzJeeFy2OPOjY=", + "summary": [], + } + ], }, - "provider_specific_fields": {}, } ], "usage": { - "completion_tokens": 5, - "prompt_tokens": 19, - "total_tokens": 24, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - }, + "completion_tokens": 135, + "prompt_tokens": 312, + "total_tokens": 447, + "completion_tokens_details": {"reasoning_tokens": 70}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, + "billing": {"payer": "developer"}, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, "service_tier": "default", - "system_fingerprint": None, + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + }, +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "openai/gpt-5.6-terra"}, + expected_provider_endpoint="/responses", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, "model": "gpt-5.6-terra"}, + mock_provider_response={ + **OPENAI_RESPONSES_RESPONSE, + "model": "gpt-5.6-terra", + "id": "resp_0b9dd9372c7413ec006ad02386760487d08468a2a84f9e6871", + "created_at": 1792025478, + "access_programs": {"cyber": "daybreak_blue"}, + "completed_at": 1792025479, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect." + ), + "output": [ + { + "id": "rs_0b9dd9372c7413ec006ad023870a4087d0a4a8f5a9ef33dd2e", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COIulgFZHAIST02Ob7jMU14QKmgY_QYY0QhjIokwGaR_TwJrp4Ipb5-_NFswljdp2HFfgxzTV24tE6ueee6YavBUQAU0IW576eke0inwgG9fpAS8jCmhEjrldcu8mjx4tMPnzltz46Mj4z9r-nsH6C4wufPbjV3dpEGOnZ4OfaJwaO_tEFAz_0a7c3PhbkHsIiBkdgN_w88NdR4_hz5A8Sk-n0z7_MN75nKnzkEM0x5xL739fjnbAxEXNMn5Ezw3ctwuwab61YzmLua32kk0BtnHqhMjEm62UOX53zCFeoD1esiY2NhB4Ceq3i88gvLBtWqvo68zprjJhplhaJ6AaOTIAPSRWjrtw0ONDTLcqRb_mwsW7vbym3LUwakUqA5qMGYke9V9XzgQ44Acv9EHgr_6f2gwdVwjeupv9v4Kn6t1omYSi3HrjDl7tVJ-Hxn8TpHxQzK2i22vjHpjldJux4OnKKN-2Dh3KgT5qFa1NfcVSex5pfbxFDjhh12acNLRsPHJ3eRAx308BI2fkG7RdQ5Qb22Ezc8MY0AxERYHxgCxFBXInVHBBxOdciNPogBLKTTQSUFz0utIGEPwTpaNx2Akanb4H0o2sHCLA12Zq0C8h7EUGT9Kd4NrTIq1hAwzCn6cQyAw1SWatwIxaD49wyBq9xgeouNaOPKEkKGQSPHbTKsFuy3qZ8or7WvsOhudVF3RCk8chzm-CF8Dxqc5a3QZzKmMrBsBNILSLx1MQrq4kmUaYwt5V_moG1jT3_c9dTq4YmmhISUHOohjXz6HylzWGXMZXKW2_skr1ypaMtnlRoO-ngjraJisUheMuJQtoubouShhm6UiLDA_352QXxQX8Or0sxqm80Pn52u2k7oG0RSu4ltmERsBayQLnsZTIns6nXnfyTVDJ64vrSiX3UBBzzkJfCWifNb5dBervyf4gzdNhtBivQnR0bhKp8gJ3YSOl0t-SzEiQ4VIL6cu6OvnMtGCQ84s6BDS8-196It4RsP7hNkxL3tVAIwgb9Euncn3tVfRxz4suPS_b-2UhlFb7rxgz0H26DxNtCprPlaawp4pk4puKno1k7G2zx_BWP6vw8mC4E0ws0bUOWl5icgdWVjZvAQn2t76Vxb9DdZaWDDN6WrNGU85DcAgSOl_LwAnwEZ0IfSuZT3Ys7os_HFqJ-Isnrsv1YtcSg1fLL9KooIWZOEI9sH_PKLIBLyZTYSardtyb2izficMFTz_cJS3tHARyY_L1sfphULymlRSBXpI0Evr-d4PW-4dxcYIBMELEklJwgze9wjeFKwHlv1iZjuGlnr6ieoZ9aJXkQuxSQ-uom3DZdAs0hoW5Xgd1h0X9Tnu9bcJKA3KBVzDMPYLHkZYRD5J1nEifcDpXryBFRBFk4yYAOrPYEuwXg__1qtipv7-IoTc3J0pVmid_q6rHFrl0-tRfqM8-sG-C6QJMahzsDccevWyntC2o-8tAsuUuA3gkmrWR3Yepsj8LydSrqGlXYi_o-rE-XwNjIkyS-rCsweT1iBl3JvWX_ZDXEoLFSC50Hevf7Ti2HmbnMFm6HoRz0kIuAC91EvcY3xWXXKOsb-owbUiYhuoOPSo3xtiHxMjLARPwymWN4LkdsiSndglL7lsJgK1fsZR-fF4QI=", + "summary": [], + }, + { + "id": "msg_0b9dd9372c7413ec006ad02387358887d091024ccc4d1075e9", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I need `shop/validation.py` next to verify the current handling of zero, " + "negative, and non-integer quantities before reviewing the corresponding " + "tests.", + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0b9dd9372c7413ec006ad0238795a087d0aa5e2e27ec92e9b8", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_6kurIo7rXnkqfZEy2J7ldPNG", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], + "usage": { + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 87, + "output_tokens_details": {"reasoning_tokens": 17}, + "total_tokens": 399, + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "openai/gpt-5.6-terra", + "id": "resp_0b9dd9372c7413ec006ad02386760487d08468a2a84f9e6871", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I need `shop/validation.py` next to verify the current handling of zero, " + "negative, and non-integer quantities before reviewing the corresponding " + "tests.", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_6kurIo7rXnkqfZEy2J7ldPNG", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0b9dd9372c7413ec006ad023870a4087d0a4a8f5a9ef33dd2e", + "encrypted_content": "gAAAAABq0COIulgFZHAIST02Ob7jMU14QKmgY_QYY0QhjIokwGaR_TwJrp4Ipb5-_NFswljdp2HFfgxzTV24tE6ueee6YavBUQAU0IW576eke0inwgG9fpAS8jCmhEjrldcu8mjx4tMPnzltz46Mj4z9r-nsH6C4wufPbjV3dpEGOnZ4OfaJwaO_tEFAz_0a7c3PhbkHsIiBkdgN_w88NdR4_hz5A8Sk-n0z7_MN75nKnzkEM0x5xL739fjnbAxEXNMn5Ezw3ctwuwab61YzmLua32kk0BtnHqhMjEm62UOX53zCFeoD1esiY2NhB4Ceq3i88gvLBtWqvo68zprjJhplhaJ6AaOTIAPSRWjrtw0ONDTLcqRb_mwsW7vbym3LUwakUqA5qMGYke9V9XzgQ44Acv9EHgr_6f2gwdVwjeupv9v4Kn6t1omYSi3HrjDl7tVJ-Hxn8TpHxQzK2i22vjHpjldJux4OnKKN-2Dh3KgT5qFa1NfcVSex5pfbxFDjhh12acNLRsPHJ3eRAx308BI2fkG7RdQ5Qb22Ezc8MY0AxERYHxgCxFBXInVHBBxOdciNPogBLKTTQSUFz0utIGEPwTpaNx2Akanb4H0o2sHCLA12Zq0C8h7EUGT9Kd4NrTIq1hAwzCn6cQyAw1SWatwIxaD49wyBq9xgeouNaOPKEkKGQSPHbTKsFuy3qZ8or7WvsOhudVF3RCk8chzm-CF8Dxqc5a3QZzKmMrBsBNILSLx1MQrq4kmUaYwt5V_moG1jT3_c9dTq4YmmhISUHOohjXz6HylzWGXMZXKW2_skr1ypaMtnlRoO-ngjraJisUheMuJQtoubouShhm6UiLDA_352QXxQX8Or0sxqm80Pn52u2k7oG0RSu4ltmERsBayQLnsZTIns6nXnfyTVDJ64vrSiX3UBBzzkJfCWifNb5dBervyf4gzdNhtBivQnR0bhKp8gJ3YSOl0t-SzEiQ4VIL6cu6OvnMtGCQ84s6BDS8-196It4RsP7hNkxL3tVAIwgb9Euncn3tVfRxz4suPS_b-2UhlFb7rxgz0H26DxNtCprPlaawp4pk4puKno1k7G2zx_BWP6vw8mC4E0ws0bUOWl5icgdWVjZvAQn2t76Vxb9DdZaWDDN6WrNGU85DcAgSOl_LwAnwEZ0IfSuZT3Ys7os_HFqJ-Isnrsv1YtcSg1fLL9KooIWZOEI9sH_PKLIBLyZTYSardtyb2izficMFTz_cJS3tHARyY_L1sfphULymlRSBXpI0Evr-d4PW-4dxcYIBMELEklJwgze9wjeFKwHlv1iZjuGlnr6ieoZ9aJXkQuxSQ-uom3DZdAs0hoW5Xgd1h0X9Tnu9bcJKA3KBVzDMPYLHkZYRD5J1nEifcDpXryBFRBFk4yYAOrPYEuwXg__1qtipv7-IoTc3J0pVmid_q6rHFrl0-tRfqM8-sG-C6QJMahzsDccevWyntC2o-8tAsuUuA3gkmrWR3Yepsj8LydSrqGlXYi_o-rE-XwNjIkyS-rCsweT1iBl3JvWX_ZDXEoLFSC50Hevf7Ti2HmbnMFm6HoRz0kIuAC91EvcY3xWXXKOsb-owbUiYhuoOPSo3xtiHxMjLARPwymWN4LkdsiSndglL7lsJgK1fsZR-fF4QI=", + "summary": [], + } + ], + }, + } + ], + "usage": { + "completion_tokens": 87, + "prompt_tokens": 312, + "total_tokens": 399, + "completion_tokens_details": {"reasoning_tokens": 17}, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, + }, + "access_programs": {"cyber": "daybreak_blue"}, + "billing": {"payer": "developer"}, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, }, ) diff --git a/tests/integration/translation/chat_completions/bases/openai_responses.py b/tests/integration/translation/chat_completions/bases/openai_responses.py index 7ae869dcc52..d086a56a221 100644 --- a/tests/integration/translation/chat_completions/bases/openai_responses.py +++ b/tests/integration/translation/chat_completions/bases/openai_responses.py @@ -2,6 +2,11 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.openai import ( + OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, + OPENAI_RESPONSES_RESPONSE, +) GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -121,98 +126,140 @@ GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "openai/responses/gpt-6.1-sol", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "openai/responses/gpt-6.1-sol"}, expected_provider_endpoint="/responses", - expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"}, - expected_provider_request={ - "model": "gpt-6.1-sol", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "stream": False, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, "model": "gpt-6.1-sol"}, mock_provider_response={ - "id": "resp_0ff994dd79332d97006ac3f04a429487d09618de316e259d7f", - "object": "response", - "created_at": 1791225930, - "status": "completed", - "access_programs": None, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791225933, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6.1-sol", - "moderation": None, + "id": "resp_0e8134ce088f63e7006ad023a2e5b887d0a0f422419e0bfbe7", + "created_at": 1792025506, + "completed_at": 1792025510, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect." + ), "output": [ { - "id": "msg_0ff994dd79332d97006ac3f04cfcf487d088bec15ca90ae36b", + "id": "rs_0e8134ce088f63e7006ad023a3ff5887d0801528ceb0190f4b", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COmpSSf5bN4H0xC2pbhfp_niGPAm58n4-JyZsHy0ytyhg6KzzTZiR0cjuNAHvZR_w8oavaY1zzH-MkWItVhAx-PQJQ560Da0Dwqi_wwJ0831R8W6lxyeFxgBZXpZ4_TsbT1qwt7IwVC6fJE4gDeFGiuo4H7plTfbE7C2DaO_0fkI5Zt_iRZLA3VOJPFnzegipK4A5AxckZNVZLI705xzMfi7KU6J886jCmsqCxjrQKLwP4fB0jQoRidBCensrctV2C7imXgTAlTwOysSd8gf7Yhu7A9qD1rUj_jvLGJCiqBCe9Cyv9I7cX82VewV4029mGOngvQOUAgVbqT_i9LkQ3HfBXu4SYTK_LXU2P0lw_P69qKYGTZfdwxqui8VVtgJI7kDrZY-l7DvqaIncKJ-LgDJBvxaYDMM7H1Co68Yd64RM3U1v4kTezKN_MBg-1Ku0m1e3Zi7YZDHOHkFDDGFXzXCpbGkgDJCHsG7i-UTE1uAjkgdP9h0MUFosMYB8pl2NxtN8NnEHsBfyYBV56Sms8ObilQaK2eEWvnDH_wkGLt2RAGOJwfovSEwKnWgOemEZtA_IYbxhRNupRJkPy0oJ4xPdAxIICncSkwaF_NMYeIetF-0tvqK2mqWZVnI13w_CPUW0WI0nlfSnlqO-7gD4Xh80-cvFAqr44X_yaRODPOOc6aWoXb_23A2AoFW6vEKufFf3zni2JEHiCREgvUAp8p931WYo7H_lDpOqIHGiQVjvRfUN9pRPnpPbF6NwCUNalgQPt4UZal_ubh3kdF1NRqKuTQMcalPfuw7Zdc7-ZFTV3JdUApBwsF2jGFXzAfmwHytnOohHOAm79n_7ORaJV3ecKjyg0t-N777eu-7LJX79tvy1tNXR-0DsodobU-mHuyxqC4Qhmqk8jC1IeNG7du1IUwpFJnOyM4O1D6xUYGslBWXsIa_exQpTkxH0PYSYeCGA9GCeWsrn8gSkOgwBmtsC56Bow8VldAr_t_AS5UIrSUsNvn1SF-nZJKbe1mvL-egWOuQq8dZb3wEF7vVxqgn4_f54pGXyDeEdxPdxAKhqT672MKb3l_K11Yn10kbhNr1XqX-m323zLcNizjPlH8rJGUWht7RGQd6VOxEjmhKC_ZWFsu7wQgqsGX0BQ2e-6hZE2Kq_9u_MfSieYJyt9jb6XXrqefSP2tyWsrrS6HrBBrSkh2d5PfiP4MQ2gTesP9tj-yVlq4JU3FZMgqS5W7m2BmhD8gGonaLSK5zJboTYjpZHoS0swfxX2_BJDiARNrPFZ58t6sSXDZauTZq8CbdLdXUBA6RaO1kp-yscPIuB2DHjPjPUVrRYbNWSPIcjqHDYY11UHPs7lWL8U5j-j3EfYwiwQ85YnVfoB7jzJFeiywAUvbum20MJXBOE2wrvrR6zXJAGYlEwxKbl_ibZULRRuJ-VSGeijiDNtavD5v5-AAmow5yPuuVUH9_kx7Un8ZbbESZH1rptCToDmxeRrIa51Xw8fIbONz1fVNzljCUvqvsoyjPQAlaZy_YfLIOYD5MVhVX8XiR8mIeSna4ELdWXu8EEmPWyl676GbSZvMzQmBVqDFeELEusHm_UNX3oI5oOMJQ8ZA9p-JyJlx8LWDe3wqTaKKzoajxn2Wz6qvjKFStkGrk0voFzEPEnBbIAIPkh7Uv2ORupzlQeF6Hor11DxlDAIaUQf1344U37C-a1ODbqL8tg7HOm1WSKdwWeKG2-SYk9K8yB1hEZuqmxHquiXybUU-EYrQFGv88d-K0ou-wAjolMdXlIUaT5-OmqA8HCprI4Nr1BdxQYLLwlNyxm5Y610a1Bd9clo8YbW-I_zSNVm74nF1_HzKP3Ym9H3_f1ZWj55BOAw9i1jmk6kb7P2SAomQ9WVCHvhbyPUhrMMlDxc5VpQOMlFRIZjCEb4wirnD4oonK5mt_TY1l_XMc7DAFb91wpdF3dB6Nh8W2_YYC8_vHfvvSq5WtVKBVIBzZM7l188EX3zHUoe4a9pbQj2Yj2-fAF4qnae4DZ1IwYvoUrln3YETLC37Qb7c3cwM_-i9YLiB_F-qA-pVnmwH8Ah2a3cOF-sl50BE7ey_rHw0fhnflc2bicoLMA82gXdE3iuAsPX-sJvouXgLB1KYmFZt_8Hm3QlrT2c8mj_Jt10YdPC_haBCvzOQDvi5kfPM2upPrvgm7bPEgyjLZECEaQj7bjgAsZc7PqnNp0u41Quq2WxUbjMrE6BPjcrhF5Llx6PQjrz2d5_023M1xBMGxKNXaLNmyi91YpvC0Sqd9qUKndUQ6fPA4z03_FP7-UnzYPuJQaBX662f1Vd-sSUvvM4zdPw6YX-muGqHF5qld2He6eMaUUXhjDuwvy2LW5WRQfSbWJ89XsUYqDwDM0j8TEexma6UZ07uNV_xjse6P8JfL-L8BDiHkR7ZNRMDSYqsHQjqwoYZNUs25XwWPr7kjn-43H0mW_dXPcManZ6qBlKKETmdtr8CZrtNxZ1_TnHUeNkATzhmtzyLQRmouxUq6tbPMumlcXcwhvDBSKSR33a4LZG1kknf_YlTDLA4x7sHImDRMYgM9d6SYxBuTSq1OfTtSv7i4Wz5_Pef3iNFI8JH6Q6b24vED1lnWvVEImDKReDYdDLoNJr2BbSt6iZ0hsGSQ5EMsE3jlizNIBwdqI5z2MKTCYWAXSbxXe62zgaklRD6IoCER9h4SqjsnOh9esKu1bg-AkSa04Le9PP4i2whspq6Xa7f4DJpRYb-hEdY", + "summary": [], + }, + { + "id": "msg_0e8134ce088f63e7006ad023a5aa0c87d0aee7de2e31f58a5c", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to see how it handles zero, negative " + "values, and non-integer quantities.", + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_0e8134ce088f63e7006ad023a615bc87d0b7aaafc731d1f086", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_sPhtuzlfkfNe16kBteKPS72V", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 312, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 166, + "output_tokens_details": {"reasoning_tokens": 100}, + "total_tokens": 478, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ - "id": "resp_0ff994dd79332d97006ac3f04a429487d09618de316e259d7f", - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "openai/responses/gpt-6.1-sol", - "object": "chat.completion", - "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello!", "role": "assistant"}}], + "id": "resp_0e8134ce088f63e7006ad023a2e5b887d0a0f422419e0bfbe7", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I’ll read `shop/validation.py` first to see how it handles zero, " + "negative values, and non-integer quantities.", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_sPhtuzlfkfNe16kBteKPS72V", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0e8134ce088f63e7006ad023a3ff5887d0801528ceb0190f4b", + "encrypted_content": "gAAAAABq0COmpSSf5bN4H0xC2pbhfp_niGPAm58n4-JyZsHy0ytyhg6KzzTZiR0cjuNAHvZR_w8oavaY1zzH-MkWItVhAx-PQJQ560Da0Dwqi_wwJ0831R8W6lxyeFxgBZXpZ4_TsbT1qwt7IwVC6fJE4gDeFGiuo4H7plTfbE7C2DaO_0fkI5Zt_iRZLA3VOJPFnzegipK4A5AxckZNVZLI705xzMfi7KU6J886jCmsqCxjrQKLwP4fB0jQoRidBCensrctV2C7imXgTAlTwOysSd8gf7Yhu7A9qD1rUj_jvLGJCiqBCe9Cyv9I7cX82VewV4029mGOngvQOUAgVbqT_i9LkQ3HfBXu4SYTK_LXU2P0lw_P69qKYGTZfdwxqui8VVtgJI7kDrZY-l7DvqaIncKJ-LgDJBvxaYDMM7H1Co68Yd64RM3U1v4kTezKN_MBg-1Ku0m1e3Zi7YZDHOHkFDDGFXzXCpbGkgDJCHsG7i-UTE1uAjkgdP9h0MUFosMYB8pl2NxtN8NnEHsBfyYBV56Sms8ObilQaK2eEWvnDH_wkGLt2RAGOJwfovSEwKnWgOemEZtA_IYbxhRNupRJkPy0oJ4xPdAxIICncSkwaF_NMYeIetF-0tvqK2mqWZVnI13w_CPUW0WI0nlfSnlqO-7gD4Xh80-cvFAqr44X_yaRODPOOc6aWoXb_23A2AoFW6vEKufFf3zni2JEHiCREgvUAp8p931WYo7H_lDpOqIHGiQVjvRfUN9pRPnpPbF6NwCUNalgQPt4UZal_ubh3kdF1NRqKuTQMcalPfuw7Zdc7-ZFTV3JdUApBwsF2jGFXzAfmwHytnOohHOAm79n_7ORaJV3ecKjyg0t-N777eu-7LJX79tvy1tNXR-0DsodobU-mHuyxqC4Qhmqk8jC1IeNG7du1IUwpFJnOyM4O1D6xUYGslBWXsIa_exQpTkxH0PYSYeCGA9GCeWsrn8gSkOgwBmtsC56Bow8VldAr_t_AS5UIrSUsNvn1SF-nZJKbe1mvL-egWOuQq8dZb3wEF7vVxqgn4_f54pGXyDeEdxPdxAKhqT672MKb3l_K11Yn10kbhNr1XqX-m323zLcNizjPlH8rJGUWht7RGQd6VOxEjmhKC_ZWFsu7wQgqsGX0BQ2e-6hZE2Kq_9u_MfSieYJyt9jb6XXrqefSP2tyWsrrS6HrBBrSkh2d5PfiP4MQ2gTesP9tj-yVlq4JU3FZMgqS5W7m2BmhD8gGonaLSK5zJboTYjpZHoS0swfxX2_BJDiARNrPFZ58t6sSXDZauTZq8CbdLdXUBA6RaO1kp-yscPIuB2DHjPjPUVrRYbNWSPIcjqHDYY11UHPs7lWL8U5j-j3EfYwiwQ85YnVfoB7jzJFeiywAUvbum20MJXBOE2wrvrR6zXJAGYlEwxKbl_ibZULRRuJ-VSGeijiDNtavD5v5-AAmow5yPuuVUH9_kx7Un8ZbbESZH1rptCToDmxeRrIa51Xw8fIbONz1fVNzljCUvqvsoyjPQAlaZy_YfLIOYD5MVhVX8XiR8mIeSna4ELdWXu8EEmPWyl676GbSZvMzQmBVqDFeELEusHm_UNX3oI5oOMJQ8ZA9p-JyJlx8LWDe3wqTaKKzoajxn2Wz6qvjKFStkGrk0voFzEPEnBbIAIPkh7Uv2ORupzlQeF6Hor11DxlDAIaUQf1344U37C-a1ODbqL8tg7HOm1WSKdwWeKG2-SYk9K8yB1hEZuqmxHquiXybUU-EYrQFGv88d-K0ou-wAjolMdXlIUaT5-OmqA8HCprI4Nr1BdxQYLLwlNyxm5Y610a1Bd9clo8YbW-I_zSNVm74nF1_HzKP3Ym9H3_f1ZWj55BOAw9i1jmk6kb7P2SAomQ9WVCHvhbyPUhrMMlDxc5VpQOMlFRIZjCEb4wirnD4oonK5mt_TY1l_XMc7DAFb91wpdF3dB6Nh8W2_YYC8_vHfvvSq5WtVKBVIBzZM7l188EX3zHUoe4a9pbQj2Yj2-fAF4qnae4DZ1IwYvoUrln3YETLC37Qb7c3cwM_-i9YLiB_F-qA-pVnmwH8Ah2a3cOF-sl50BE7ey_rHw0fhnflc2bicoLMA82gXdE3iuAsPX-sJvouXgLB1KYmFZt_8Hm3QlrT2c8mj_Jt10YdPC_haBCvzOQDvi5kfPM2upPrvgm7bPEgyjLZECEaQj7bjgAsZc7PqnNp0u41Quq2WxUbjMrE6BPjcrhF5Llx6PQjrz2d5_023M1xBMGxKNXaLNmyi91YpvC0Sqd9qUKndUQ6fPA4z03_FP7-UnzYPuJQaBX662f1Vd-sSUvvM4zdPw6YX-muGqHF5qld2He6eMaUUXhjDuwvy2LW5WRQfSbWJ89XsUYqDwDM0j8TEexma6UZ07uNV_xjse6P8JfL-L8BDiHkR7ZNRMDSYqsHQjqwoYZNUs25XwWPr7kjn-43H0mW_dXPcManZ6qBlKKETmdtr8CZrtNxZ1_TnHUeNkATzhmtzyLQRmouxUq6tbPMumlcXcwhvDBSKSR33a4LZG1kknf_YlTDLA4x7sHImDRMYgM9d6SYxBuTSq1OfTtSv7i4Wz5_Pef3iNFI8JH6Q6b24vED1lnWvVEImDKReDYdDLoNJr2BbSt6iZ0hsGSQ5EMsE3jlizNIBwdqI5z2MKTCYWAXSbxXe62zgaklRD6IoCER9h4SqjsnOh9esKu1bg-AkSa04Le9PP4i2whspq6Xa7f4DJpRYb-hEdY", + "summary": [], + } + ], + }, + } + ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 19, - "total_tokens": 25, - "completion_tokens_details": {"reasoning_tokens": 0}, + "completion_tokens": 166, + "prompt_tokens": 312, + "total_tokens": 478, + "completion_tokens_details": {"reasoning_tokens": 100}, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0}, }, "billing": {"payer": "developer"}, diff --git a/tests/integration/translation/chat_completions/bases/vertex_ai.py b/tests/integration/translation/chat_completions/bases/vertex_ai.py index 23876f4a129..21c6b8a0072 100644 --- a/tests/integration/translation/chat_completions/bases/vertex_ai.py +++ b/tests/integration/translation/chat_completions/bases/vertex_ai.py @@ -2,13 +2,18 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.vertex_ai import ( + VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, + VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_0, + VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_1, + VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, +) GEMINI_3_5_FLASH_THOUGHT_SIGNATURE: Final = "AY89a1+kriNs9YsPegJQxgjTWsMlxsqPU/UkHDyz4EK+3FRrE52ygqPIHTvkblgKKiO5Fn8R9D3iu//i+YnuuZ+e7W0yxUoFScas89b5rn3vZHsihnUA7VTe3b7beth1IftteoTGZ3F8OSLrSIP8HKn4xTIx35eeNEt5oiD3K5rTQrdklqO277HvTDhu/LuZhZ0pzaWciGQEc61epIBBpC2ygu8V4L8TekUVZnz6qzSzxipdiQStQ8dZou43334hMnxOR6JjJCGxqt/1K+C0vXX6YP6f+Bph5VhPSnvy2Z1uApi3TUxAcsDDyVewIr+ha2G6DPhN1Oi5uZZNhmbvH01Uq5OOquF1uEICq2m/35LhJIbDNM9N7oI1Z6L94P9xdugeFXM2S7wdJfc8sdg09vEzNeA++GK25zRuB4hU7XyaG4XnWjmu2kswi4sf+EyrO/K0P2tb/ZCJu7kJTtWTiD1wgSIl1eqS8ORKKsXDN2hIbFrtsNR50WIvVzd6" -GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = "AY89a1+gmp4bVJhwGOCtbUalmyfhwvgE4EsZOwS4bNR02vurTt4AZC72ypTNLJDRwK8I518tGYBiDnFqD6jTxkjWTQTPGwUZ5IOYr1IskTb3pPKfvZWZ/XunV7kfb9VmEU0TBlgwjVqxXziOxvjediJyb2gbFnCVjGTSVsq0ipmpFznF22lV1I6N9GI7RKBFgYXaSzGU1+fGbLLBigaUD0qbgquAnFv6R3JGpaLcW7EUrhKDinHoFukmUwBLx07S7SfuWBT8hMLMs32vEVzwpns9JmbVtuQG24zs5qQx9sJqtO9E07k6SFL3f1tvaKZsQ6Yjb08Xcuvnh9Pcz4ChCdRJ4vRkmtRW3AgOli8/43rrrAMmvIOqEdAOOvZW3W9D8lnOC+Nfq7S0FTNWPYkIJhrtskWoQkUwDrY6TOzMjIA6Jblt2f9kKU3VJzq7O1hPjbHsJKmLBYP+De8rKx7Ufq9Shs5qLLu4aBBgHleiyFiHGQSD18RDjnyv0DLRyrp7LOkjIYWBYoIA2+BSb0ySMyeLhk8B2U8XZ6AXiw3aHoNUG/wwuE0EhXBhOUBM14FmchGfptXUr+Dn9kZvPAy8mfbq3Pk=" - -GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = "AY89a198sdR5Cy3qgtvOq/o5xsmH1g3WymzyZyB/bv3XeyqNZAgXbCocCAqUS913U8FvVUodNrm3y2B5P/mDsn3A3pINhbEiwJcWA/o8bNJq+Z/1O2UYdViO0FMhOOS1aRaCbza+6aSBLPVQ81QnyxrwH6rdDCuQlwyZ+5/Evc/EOXLVxdUF9PYwTwNJcZPREl4UMcarKvtKsNofkSfa09EWQNVVMDkasNl3YHtQJqYLu5rL7vSUrbMtLLA0sMEMh2/4zMj4gIvtjoOwiI4CBdLLH0Xp8SGGco8UljxikaIp6YvQW4cHsfkaqTBQmLgREudbnKKTZ0MvMR52G/++7r8Iv6BL29+73igMMyHoQad40OM8RY3E6akpSE5uPXH/CgYTmSLXQONtRMXzrXm996JYLaHLOo2dP1RtaJANcjNoDB4eN5vj4GOpTUYqZqRRaB5jMX2od8QcAnaXk2AYs/2rNMaLV4zgraz45gbA0It465nN5hb4JbRH2toJ9K/TvsZY/YyoLmV9RjyAmIhbq6bG0fGatsXgmDaXj/vEsBmqfRyBN31k" - CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", @@ -323,18 +328,139 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( +GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", litellm_request={ - "model": "vertex_ai/claude-sonnet-5-5@default", - "max_tokens": 64, + "model": "vertex_ai/gemini-3.5-flash", + "max_tokens": 1024, "messages": [ {"role": "system", "content": "You are a terse assistant."}, {"role": "user", "content": "Say hello."}, ], "cache": {"no-cache": True}, }, + expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.5-flash:generateContent", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, + expected_provider_request={ + "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], + "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, + "generationConfig": {"max_output_tokens": 1024}, + }, + mock_provider_response={ + "candidates": [ + { + "content": { + "role": "model", + "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_5_FLASH_THOUGHT_SIGNATURE}], + }, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 9, + "candidatesTokenCount": 2, + "totalTokenCount": 95, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], + "thoughtsTokenCount": 84, + }, + "modelVersion": "gemini-3.5-flash", + "createTime": "2026-10-05T22:54:50.809002Z", + "responseId": "uirEaqqwMc-U9LsPjJTYuQU", + }, + expected_litellm_status_code=200, + expected_litellm_response={ + "id": "uirEaqqwMc-U9LsPjJTYuQU", + "created": ANY, + "model": "vertex_ai/gemini-3.5-flash", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello.", + "role": "assistant", + "images": [], + "thinking_blocks": [], + "provider_specific_fields": {"thought_signatures": [GEMINI_3_5_FLASH_THOUGHT_SIGNATURE]}, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + } + ], + "usage": { + "completion_tokens": 86, + "prompt_tokens": 9, + "total_tokens": 95, + "completion_tokens_details": {"reasoning_tokens": 84, "text_tokens": 2}, + "prompt_tokens_details": {"text_tokens": 9}, + }, + "vertex_ai_grounding_metadata": [], + "vertex_ai_url_context_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_citation_metadata": [], + }, +) + +CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_0: Final = ( + "CAQS2AUKEggSEAIYAjgBQgh0aGlua2luZxIMARf7mDY/EfIxNI90GgyxpPEOoCV8iuAHRDwiMEya9+X1kJMbECIvAPWkwThHF5lM" + "isFNo67kPmn/ek1/pKWvb6bc8EzcFYnNOyBtdCrzBB7ZgNYNQvTVWJrNQAET5gFlr7MeN5oNfe4tnrSysPx7f3rCzYfsUMswfE8O" + "4sAJk87znpkMO9QRwq+PtgXijy8wWo4ERbvvKh1rPfgDhMSKkiujiaKJgMvMMM8NltTzpaaWlaFcWJNwG/k03dyJHhtvmwhaJyIP" + "+W8cLd8iVAzHc8KtYlnXOIqpFOeto9p6rRK2NgykT1gZEqCkiFs89qnOv8S0Z3RWwWdMype2EhhFcNI+jWl52OzM/kbZxtnhQ+32" + "cuntmmgYGQOXUujuAIZwmE4Rvs4U5j5sGjTE5zJZBqZUCtGBUJ9aQv6nVHGqwdy8dmpMexKPhOw/Uccq5nK3/WrufYrSFWuxz7A1" + "egu/dURaBy1+9T7j0G5VYIeXwsTOh1uJuSk2pgg+dI+9/gk5Lbt1K14ghBxIDvHEfikoAqyGhVBIS1Wn5qIFkwOowwm0S+z7x/s+" + "/3zy0gQoiEddpn52HkWzKKwrN5qZXJKmsn1pzvdQXyghGSYc5EL4BWZNhvccQgEMcJamK3+QyTGbf7mJM+D4yQNkdRAkurpNqM2O" + "ofMSaOIXRlhN5zqjNGK5bCSO+2BL0HTaRbsRYDq1jVL+DXLyDymQ1QrXWssmgVo0k3Mzfkj1MenT77LCV2/ar0SYX+6GhBJlHm/B" + "WmNh0rE8yq8SJY7A+4mXqg+4a0bBoMa7gHBNxTI4x3XBm75ctVkAieHVKORgM23sCQFvngXql0WTZUOC0QHf54MTGOoNBJYIJP7H" + "Xu2RKL6lL9eqQFNYVEDQK/NyWHX2bw51g0TItCaW3Rl3T6YWXKlcVvAUBH8OQNjQII0j3yF+oGd6mxgB" +) + +CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_1: Final = ( + "CAQSgAYKEwgSEAIYAjgBQgluYXJyYXRpb24SDNLS/zBzyAtsyJk3zBoMp+HCAq92FqxGZNofIjCv16GJ3WIGycSbcFtkGKdAUa4m" + "GzcV7kH27GOyo/OLT/1GKuFrMp35qn1Jo1EvMBwqmgXZS6p/Jn0PHQj1KEteqdiVwW3Dt6t+vSPrGvq0Ned/d1q62jwgfdWU3xXj" + "/WkbifL+c3Z1tnsLLtQw5/6FuYlD4NmxypiTWq7MmOrbQ6B5oV5nM9xd86Gqrc9mkSrptbRWyQOUQZFuafCXtzAThnAdaitbS/4I" + "NvHyTgYwt096mDJB8p/aYQQlfu7bKjmzIwBxibfMRE8EzrmLP9Lnwjo+tU2qFdW/jn96SYW7u8vSmochwPeCLmXWjBwABenlhSsY" + "ncUrohBqFrfGZ0eoOOltvI7CFIKedDyZEVWgj9k7/fWTg5FZuAYUXTj/NMka1CDe05zAReAqW/0qOVw9XLlqQ/+84A+hQnrdggTf" + "24KGSmgHJ0gkmLNSnXgzSZd374QaNphAzUJy4MOMoYzu8KIrRev2FE05cDChzCQm3N27gL8oiqZzONNxYsNgbUu/30rMyvRla1uA" + "e/a9nhXAfvGk0zXoh+cprhGOf9FmVWHg7jrnn7s3ZumAUDKpckvoUyiXTd2raFsOXd/c8fZNaZTyz28M0mfDDbMtWJdNqQLpJJLc" + "Xr1YTJ7HNZeeA4TZiOp4CX4iEnI+RYylaXGLIodHCCOosriu9EHeceMIHASB+i6lxcL8OUUkTbASFbpe7qHB0wsqwDJGUO35fdJa" + "mKWY9gM5GEMORkkeVzqT1CsELducpP58h54yC1kEl3z31qko5No2d7Tsb8C0KNAm2sr8O20cOGLxZ9hzAE4e0WQSF0gFqRQ3xAX9" + "yoUU5tw4Q7PygHCzyWMArMf+GhJfoZZfOrvUl+KYu73xCshFSmi1OUft29RQRLZ9DRNagtwdro4LcBf2jVTzOZcUJ0VWl4J0ebuF" + "yc4V9fnxZTe4KTHLr2XV5+qytgDzk4MYAQ==" +) + +GEMINI_3_1_PRO_PREVIEW_CHAT_COMPLETIONS_THOUGHT_SIGNATURE_0: Final = ( + "AY89a18NmP/rpOzYqPT2gXtTGPJwCXopurRCh2SERSxHAbUcM+kBs62qrKZf6X3eHEdd/leFa2/wBviqJ6WkkEdIsgz94SdkfwWm" + "53Pd6IC20z+9KTq6PpsXs5CJQzMRyWvSBBolC4JHE99knIqr4nJbUlXHYCknKnExuUNUuWzkcY2zsO/uX81og8IwGDikReqErvvU" + "+EVyAV0TSDb3Pg0A8eC7RCHBLPfT0ljwzNJDibRUkwoxdKcPkMvdbYhZEAnkOPD7N/CpITooxxwBYVArsbDW1oA4kL78A2gnbsNi" + "Y2esZSaKgxrftJk1H06rW5b7tAk7y4wMjbj7GD9XQJJ3HzLdUznXNvLRZjkNcr8evFgKPq3YIPqpuCn5klH09/qjGNZRkKoYriBS" + "f95iWKvGZ7jwEHu+45Qw46o0lID1TFZsrnS4WiwllgqAhvvFZ//DbCtpjMIztpTGYD754oc60k50p6Ny14oPDBlNMOVqwiFGq9QC" + "FQ7Vzilx+rqAoVdOzvyGSf6+SfXOT0Jv/raw+HIIA2bhOqurVqgdS4CoxdpCcZ5CmOgN4CIA9rQcuxZ/dVaT2w/UL0fUJOAN3kSU" + "ngcI+FcqIjlbF+IbIEGphpBhsIaTd3qXVCH/57K0tP7lVu9GoAU9zNu/6YPIWZnE9ABUBfQkyTR/DdFZm3Hfsjt7m5YkPlm+kJNH" + "iHHwdjTo+liy9gBFnQbl+mFgv3e0bcdy+a0pa6mUyluy0IWrKFBVFcfEep4vD7+CRTpkpwhwzL6+v2t3LWGcJGZV6GS/qbbGR3ys" + "Cyr70DrZtaEnbOnLN3UYP6zDcDfKYTn7ub3hIjaME9LNjg==" +) + +GEMINI_4_ARGON_CHAT_COMPLETIONS_THOUGHT_SIGNATURE_0: Final = ( + "AY89a19UcUynGbcdZOu5ucJyuVTEBkP7BWc+WQRD+zdel8qY2zqNh1OLS6DejBKWK9xndkK8coxMqPRuuQytzyB+zVvJz0ouSNh0" + "nH3Bm0p+S9hFJDzNhlHrk4PRtU0OWxv5SKWtzQURmeBDk5mJV/MU9yJSDs7g+csm6juZi2Lang76vPWyQOHTVPYnJVzLHkP73H0D" + "WnVPXSdNzBgqPzopqf3GoP+EvALphWyi0Zp/1uv7m4TKbx9dRZjlO15LaqpksnkDvwkF6o8KfaYrC/Skh+xYxlk40yxiiaYv5sr4" + "PbGutzCGNWCZxjbBCbQBPGMFIL3fDGwLavbGd71tWKcOaSBr3oYbsw5HsfWMFHV8+TQ7rou77MCMWaiJ3mulflSmdBkkTq0m1bPm" + "iIbwYvVPs6CCtTTZgGYTZhRefefwD1Dx4Y5epmGy1+jST7zikArzdZeexaCTTrIWO4rvqzPdAS65XHnfXjgYEPYK0YUnX6StK/ln" + "Lrwdb+AK6j+Q2eLuKpx/HJE0QDZ9BgXpw/CQ6my0s2rI3Tjlbun8Bdl7MyFuCbstbTA+uu1RXXv2M1RThJQHiuDGvgzCNg/sgyCb" + "zluzK/vp2nzki+ESsFeForBBSSCLVAep2tiqKhIOWPWBHYjH04qfgyWkylbRbZeA32+BPYViEcZa+Imi+aYXu241RDOSWRMJvbsJ" + "oDY15ThjQ+sgLNKJkBoravc7+sQ1Abt+jwCcO9zHBquTWn8G/X0IggqHNuDnbWEW3S9HdxPXw7ZV0ZYXXkZXIYwFIt6Ji7LpXlEY" + "X0hOwrBdq2ufKAv+EWBQyCU3oyBrIGeVdVx25BiLFsVJzqw3xLTUgfelHyawgwtGvd7J+mQBPp4YH1Rk6fbQMwDEWipSiltG+jG9" + "ZqXUiF1NtN30tw4a4h2tA03uBV1yXhv6+fL7g2zmN6c4GVX9IzAS60UUnXIMlBgCXjxCU591TuPufe6mJQ==" +) + + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "vertex_ai/claude-sonnet-5-5@default"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-sonnet-5-5@default:rawPredict", expected_provider_headers={ "authorization": "Bearer scripted-token", @@ -342,78 +468,109 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", "x-api-key": "scripted-token", }, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "anthropic_version": "vertex-2023-10-16", - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, - mock_provider_response={ - "model": "claude-sonnet-5-5", - "id": "msg_vrtx_011CfjuywExcu3BdZ3x1KzVJ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - }, - }, - expected_litellm_status_code=200, + expected_provider_request=VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + mock_provider_response={**VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-sonnet-5-5"}, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "vertex_ai/claude-sonnet-5-5@default", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_vrtx_01SYkCZM7zmQWqcyokJjiAyf", + "type": "function", + }, + { + "index": 3, + "function": { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 50}', + "name": "read_file", + }, + "id": "toolu_vrtx_01UhxynEUZYnaNhRkRGC2ZGn", + "type": "function", + }, + ], + "reasoning_content": ( + "The user asked to read the validation file first, but since reading the tests is independent, I " + "think it's fine to read both files together rather than strictly sequencing them.\n\nI'll check " + "`shop/validation.py` for how `validate_quantity` handles zero, negative, and non-integer values, " + "and review `tests/test_validation.py` for the expected behavior.\n\n" + ), + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "The user asked to read the validation file first, but since reading the tests is independent, I " + "think it's fine to read both files together rather than strictly sequencing them.\n\n" + ), + "signature": VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `shop/validation.py` for how `validate_quantity` handles zero, negative, and non-integer " + "values, and review `tests/test_validation.py` for the expected behavior.\n\n" + ), + "signature": VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_1, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "The user asked to read the validation file first, but since reading the tests is independent, I " + "think it's fine to read both files together rather than strictly sequencing them.\n\n" + ), + "signature": VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `shop/validation.py` for how `validate_quantity` handles zero, negative, and non-integer " + "values, and review `tests/test_validation.py` for the expected behavior.\n\n" + ), + "signature": VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_1, + }, + ], + }, + "content": None, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 425, + "prompt_tokens": 880, + "total_tokens": 1305, + "completion_tokens_details": {"reasoning_tokens": 229, "text_tokens": 196}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, }, }, ) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/claude-opus-5-5@default", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "vertex_ai/claude-opus-5-5@default"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-opus-5-5@default:rawPredict", expected_provider_headers={ "authorization": "Bearer scripted-token", @@ -421,307 +578,136 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", "x-api-key": "scripted-token", }, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "anthropic_version": "vertex-2023-10-16", - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, + expected_provider_request=VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, mock_provider_response={ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-opus-5-5", - "id": "msg_vrtx_011Cfjuz6qugj8FbC3fcXdun", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - }, - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "vertex_ai/claude-opus-5-5@default", - "object": "chat.completion", - "choices": [ + "id": "msg_vrtx_011Cg38nZLrXXSDtDRwwmsco", + "content": [ { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, - } + "type": "thinking", + "thinking": ( + "I'll focus on reading validation.py around line 18 first, since the user specifically asked for " + "that file before anything else.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I found `validate_quantity` at line 18 in `shop/validation.py`, so I'll check the surrounding code " + "to see how it handles zero, negative, and non-integer quantities before looking at the tests.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_0128Am9iqNFAAAMfU7bBdpQz", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 45}, + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "input_tokens": 4, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 265, + "output_tokens_details": {"thinking_tokens": 167}, + }, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "vertex_ai/claude-opus-5-5@default", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 45}', + "name": "read_file", + }, + "id": "toolu_vrtx_0128Am9iqNFAAAMfU7bBdpQz", + "type": "function", + }, + ], + "reasoning_content": ( + "I'll focus on reading validation.py around line 18 first, since the user specifically asked for " + "that file before anything else.\n\nI found `validate_quantity` at line 18 in `shop/validation.py`, so " + "I'll check the surrounding code to see how it handles zero, negative, and non-integer quantities " + "before looking at the tests.\n\n" + ), + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "I'll focus on reading validation.py around line 18 first, since the user specifically asked for " + "that file before anything else.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I found `validate_quantity` at line 18 in `shop/validation.py`, so I'll check the surrounding code " + "to see how it handles zero, negative, and non-integer quantities before looking at the tests.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_1, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "I'll focus on reading validation.py around line 18 first, since the user specifically asked for " + "that file before anything else.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I found `validate_quantity` at line 18 in `shop/validation.py`, so I'll check the surrounding code " + "to see how it handles zero, negative, and non-integer quantities before looking at the tests.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_THINKING_SIGNATURE_1, + }, + ], + }, + "content": None, + }, + }, + ], + "usage": { + "completion_tokens": 265, + "prompt_tokens": 880, + "total_tokens": 1145, + "completion_tokens_details": {"reasoning_tokens": 167, "text_tokens": 98}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, }, }, ) -GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/gemini-3.5-flash", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.5-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_5_FLASH_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 95, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 84, - }, - "modelVersion": "gemini-3.5-flash", - "createTime": "2026-10-05T22:54:50.809002Z", - "responseId": "uirEaqqwMc-U9LsPjJTYuQU", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": "uirEaqqwMc-U9LsPjJTYuQU", - "created": ANY, - "model": "vertex_ai/gemini-3.5-flash", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_5_FLASH_THOUGHT_SIGNATURE]}, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - } - ], - "usage": { - "completion_tokens": 86, - "prompt_tokens": 9, - "total_tokens": 95, - "completion_tokens_details": {"reasoning_tokens": 84, "text_tokens": 2}, - "prompt_tokens_details": {"text_tokens": 9}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, -) - -GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/gemini-3.8-flash", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.8-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 118, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 107, - }, - "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-05T22:55:12.739855Z", - "responseId": "0CrEao-ULbuU9LsPk-mQ2A8", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": "0CrEao-ULbuU9LsPk-mQ2A8", - "created": ANY, - "model": "vertex_ai/gemini-3.8-flash", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_8_FLASH_THOUGHT_SIGNATURE]}, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - } - ], - "usage": { - "completion_tokens": 109, - "prompt_tokens": 9, - "total_tokens": 118, - "completion_tokens_details": {"reasoning_tokens": 107, "text_tokens": 2}, - "prompt_tokens_details": {"text_tokens": 9}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, -) - -GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/gemini-3.1-pro-preview", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.1-pro-preview:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 1, - "totalTokenCount": 99, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 1}], - "thoughtsTokenCount": 89, - }, - "modelVersion": "gemini-3.1-pro-preview", - "createTime": "2026-10-05T22:55:14.874538Z", - "responseId": "0irEaqqwNbCdq8YP3L2mqQ4", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": "0irEaqqwNbCdq8YP3L2mqQ4", - "created": ANY, - "model": "vertex_ai/gemini-3.1-pro-preview", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE]}, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - } - ], - "usage": { - "completion_tokens": 90, - "prompt_tokens": 9, - "total_tokens": 99, - "completion_tokens_details": {"reasoning_tokens": 89, "text_tokens": 1}, - "prompt_tokens_details": {"text_tokens": 9}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, -) - CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/claude-fable-5-1@default", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "vertex_ai/claude-fable-5-1@default"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-fable-5-1@default:rawPredict", expected_provider_headers={ "authorization": "Bearer scripted-token", @@ -729,210 +715,259 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", "x-api-key": "scripted-token", }, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "anthropic_version": "vertex-2023-10-16", - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, + expected_provider_request=VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, mock_provider_response={ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_vrtx_011Cg346VGTLVWPwvR6aKfy7", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_vrtx_011Cg38nKaNiEJbdoCePiCCB", + "content": [ + { + "type": "text", + "text": ( + "I'll read the validation function and the zero-quantity test together, since neither depends on the " + "other." + ), + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01B9TTi6KdxKcFt6W9mhsdWT", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01XwRiPmJA1nhRaNgijpqU97", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 226, "output_tokens_details": {"thinking_tokens": 0}, }, }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": ANY, - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "vertex_ai/claude-fable-5-1@default", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", + "content": ( + "I'll read the validation function and the zero-quantity test together, since neither depends on the " + "other." + ), "role": "assistant", + "tool_calls": [ + { + "index": 1, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "name": "read_file", + }, + "id": "toolu_vrtx_01B9TTi6KdxKcFt6W9mhsdWT", + "type": "function", + }, + { + "index": 2, + "function": { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}', + "name": "read_file", + }, + "id": "toolu_vrtx_01XwRiPmJA1nhRaNgijpqU97", + "type": "function", + }, + ], "provider_specific_fields": {"citations": None, "thinking_blocks": None}, }, - } + }, ], "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, + "completion_tokens": 226, + "prompt_tokens": 880, + "total_tokens": 1106, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 226}, "prompt_tokens_details": { "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, }, - "cache_creation_input_tokens": 0, + "cache_creation_input_tokens": 876, "cache_read_input_tokens": 0, }, }, ) -GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/gemini-3.5-flash", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.5-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_5_FLASH_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 95, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 84, - }, - "modelVersion": "gemini-3.5-flash", - "createTime": "2026-10-05T22:54:50.809002Z", - "responseId": "uirEaqqwMc-U9LsPjJTYuQU", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": "uirEaqqwMc-U9LsPjJTYuQU", - "created": ANY, - "model": "vertex_ai/gemini-3.5-flash", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_5_FLASH_THOUGHT_SIGNATURE]}, - }, - "provider_specific_fields": {"native_finish_reason": "STOP"}, - } - ], - "usage": { - "completion_tokens": 86, - "prompt_tokens": 9, - "total_tokens": 95, - "completion_tokens_details": {"reasoning_tokens": 84, "text_tokens": 2}, - "prompt_tokens_details": {"text_tokens": 9}, - }, - "vertex_ai_grounding_metadata": [], - "vertex_ai_url_context_metadata": [], - "vertex_ai_safety_results": [], - "vertex_ai_citation_metadata": [], - }, -) GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/gemini-3.8-flash", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "vertex_ai/gemini-3.8-flash"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.8-flash:generateContent", expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 118, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 107, - }, - "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-05T22:55:12.739855Z", - "responseId": "0CrEao-ULbuU9LsPk-mQ2A8", - }, - expected_litellm_status_code=200, + expected_provider_request=VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + mock_provider_response={**VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, "modelVersion": "gemini-3.8-flash"}, expected_litellm_response={ - "id": "0CrEao-ULbuU9LsPk-mQ2A8", - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "vertex_ai/gemini-3.8-flash", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", + "content": ( + "To see how quantities (including zero, negative, and non-integer values) are validated, I need to " + "read `shop/validation.py` next." + ), "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": ( + "AY89a1/K8QT9La7l3gDc5jImwnXSF0xUuyg8rAUnAyYS4c2/Yjd/fV3Ve1tHSww5bSooGMdZ0oqljadXXxUqYi+7fkKOWD0Ij80K" + "LFu1O//r13u8cIpIqUxEi3lNiEn8rK6WVW5DPTMTBddZBL+/DcpCEi7U3m5vGYfoJUm9AJwQB8XDRdqLdA1hfiRnCqGpLJsFAQ2g" + "Ln3FpQUPR3+FRoWRlJnNRjk8vWaq3mBKHSHttLMRd3rpvo6VAAZbs2OGoBJXFXA9S+RyN532kDK3fZIGGW2Fk+FSD9EJ1hN1+Qeh" + "v8gzcxyZOin2O/ONFWEQ4zf8lHTzaostuIMLkKFM3v/ssK+Y4VgS0sPFr7VFByhczwoDx73oh141iDD5ch7tqAgnR7YikrWQDuV2" + "8FBSB7+MQ3BrBSt6XUvYgcHsGNbg0giE3D3pFDR7uvLfvgjhFFdIqnCWWmPfBgZ7GOD55unG0IUPA2udP65xg88r8TjfpQOUsOnN" + "KZLOiKovzHrXZX2L0tRGha5nE8ICi427Mc5YF9lm0Ilpo/BrGuuusMG3Zx9RxRyjRa9+zWPANRrvuq7SOVnFvAss4uu9PYf5Orwn" + "nGFJ0xjP1hn22yXOxRYy62ySQQ1ValpWXjPPXvGKZRiUArCuvrBvDEZoUfJhlcVXs26kYnqRikRLev4GZGvBIDnGHyWLvMo0gx4r" + "1kBnAGl5WyfZJu5SFywm/o5fMP70zVUoVdY+zC885uqB5pEV+VsAMyy98NumfmBF7kGtIjbt4l+hAqpzM1ME1z0kRqN7n9JEKn0H" + "DlsQAD9V582O2bBVSTRzF8aFJ7RKohGhrFqyGmYLA+ZAv5Z4kJ1HBZg1LEz2G88MT2vznP6lCEH0m9da48Q5/85pNQ5dlGGEkhpP" + "I+xazh6EZ0cFNGdNY1vAQY5rBQ8LPwizNSKenYvqwMdobWy8xReIQWCsOk2l4QCtlu8kC+iabWrsCPOfjdQmYTlpI+emQwssWXYU" + "UQ+wDGl7QWoaSAnwGu+XQMQ9ndaUycPdBu5+cuw815uiNNtIVfeKejFIbLF66Jq/duKfZ9tgWthPzpoM50u9aie2FZwNOuEtaLIv" + "WcdtBpY9LvnAemSuDUTLLSuiQYwBwRq8HKdFARJ362Wf8gg+ldoeXEV9R6+cscPaeY5j/+E/rsTWyxixg614qiFQyF5xL/7DWjUI" + "LBuIrmJUIVcu8LelTUslepTGF6CBTJ0RABv8pK7S8dUDy+HXtvT8bmz8l9zOsaTagC1jUpvaeGpi44gorN9KU7z56FMxoKSeKF7M" + "B2zuTFBN/GekI9gddLfQRaVK/9LAD4j+uBzdt/XICe2kGqjNu3ds9tnx82/9beezRyTRfFQ+p1V/jfaEt8AMTZ5I2IZQAkgkAnXa" + "KieRKs8qKbkF5yjhtI0krsyg4+BDQZzog+dgkxqYxLDuNyd6hN9HlnMEBAwW8hxWGp4WE8GvL2IU529JVMaBvK/NcrGNnPW/m158" + "BwaIFSfbSxQbqjv6KKqMi30i2bowXBAJit2YQHb3VN1reFBynM9mRm71PGLAbizdVS5VIqNI5zGquZrgpmTqqMF/AjsWpVSTo6Gi" + "WD3O2PJWReehkH3aMJMFeu3jv0/Ru+Ihw1zwFcZr0Yh7rK2NEYQyZjcmf5/vX0rBh89HEVdc78j16arXcFWHNZS2lLAm0NC+Xmak" + "WIP3YUcba6POPYKwqP2huUccyyGg+qFPTcWF488PNm4H86YQRrnUSkPEofjMuaLMrp3vS1TgpjuUZRiZ" + ), + }, + "function": { + "arguments": '{"start_line": 1, "end_line": 50, "path": "shop/validation.py"}', + "name": "read_file", + }, + "id": ( + "call_267302__thought__AY89a1/K8QT9La7l3gDc5jImwnXSF0xUuyg8rAUnAyYS4c2/Yjd/fV3Ve1tHSww5bSooGMdZ0oqlja" + "dXXxUqYi+7fkKOWD0Ij80KLFu1O//r13u8cIpIqUxEi3lNiEn8rK6WVW5DPTMTBddZBL+/DcpCEi7U3m5vGYfoJUm9AJwQB8XDRd" + "qLdA1hfiRnCqGpLJsFAQ2gLn3FpQUPR3+FRoWRlJnNRjk8vWaq3mBKHSHttLMRd3rpvo6VAAZbs2OGoBJXFXA9S+RyN532kDK3fZ" + "IGGW2Fk+FSD9EJ1hN1+Qehv8gzcxyZOin2O/ONFWEQ4zf8lHTzaostuIMLkKFM3v/ssK+Y4VgS0sPFr7VFByhczwoDx73oh141iD" + "D5ch7tqAgnR7YikrWQDuV28FBSB7+MQ3BrBSt6XUvYgcHsGNbg0giE3D3pFDR7uvLfvgjhFFdIqnCWWmPfBgZ7GOD55unG0IUPA2" + "udP65xg88r8TjfpQOUsOnNKZLOiKovzHrXZX2L0tRGha5nE8ICi427Mc5YF9lm0Ilpo/BrGuuusMG3Zx9RxRyjRa9+zWPANRrvuq" + "7SOVnFvAss4uu9PYf5OrwnnGFJ0xjP1hn22yXOxRYy62ySQQ1ValpWXjPPXvGKZRiUArCuvrBvDEZoUfJhlcVXs26kYnqRikRLev" + "4GZGvBIDnGHyWLvMo0gx4r1kBnAGl5WyfZJu5SFywm/o5fMP70zVUoVdY+zC885uqB5pEV+VsAMyy98NumfmBF7kGtIjbt4l+hAq" + "pzM1ME1z0kRqN7n9JEKn0HDlsQAD9V582O2bBVSTRzF8aFJ7RKohGhrFqyGmYLA+ZAv5Z4kJ1HBZg1LEz2G88MT2vznP6lCEH0m9" + "da48Q5/85pNQ5dlGGEkhpPI+xazh6EZ0cFNGdNY1vAQY5rBQ8LPwizNSKenYvqwMdobWy8xReIQWCsOk2l4QCtlu8kC+iabWrsCP" + "OfjdQmYTlpI+emQwssWXYUUQ+wDGl7QWoaSAnwGu+XQMQ9ndaUycPdBu5+cuw815uiNNtIVfeKejFIbLF66Jq/duKfZ9tgWthPzp" + "oM50u9aie2FZwNOuEtaLIvWcdtBpY9LvnAemSuDUTLLSuiQYwBwRq8HKdFARJ362Wf8gg+ldoeXEV9R6+cscPaeY5j/+E/rsTWyx" + "ixg614qiFQyF5xL/7DWjUILBuIrmJUIVcu8LelTUslepTGF6CBTJ0RABv8pK7S8dUDy+HXtvT8bmz8l9zOsaTagC1jUpvaeGpi44" + "gorN9KU7z56FMxoKSeKF7MB2zuTFBN/GekI9gddLfQRaVK/9LAD4j+uBzdt/XICe2kGqjNu3ds9tnx82/9beezRyTRfFQ+p1V/jf" + "aEt8AMTZ5I2IZQAkgkAnXaKieRKs8qKbkF5yjhtI0krsyg4+BDQZzog+dgkxqYxLDuNyd6hN9HlnMEBAwW8hxWGp4WE8GvL2IU52" + "9JVMaBvK/NcrGNnPW/m158BwaIFSfbSxQbqjv6KKqMi30i2bowXBAJit2YQHb3VN1reFBynM9mRm71PGLAbizdVS5VIqNI5zGquZ" + "rgpmTqqMF/AjsWpVSTo6GiWD3O2PJWReehkH3aMJMFeu3jv0/Ru+Ihw1zwFcZr0Yh7rK2NEYQyZjcmf5/vX0rBh89HEVdc78j16a" + "rXcFWHNZS2lLAm0NC+XmakWIP3YUcba6POPYKwqP2huUccyyGg+qFPTcWF488PNm4H86YQRrnUSkPEofjMuaLMrp3vS1TgpjuUZR" + "iZ" + ), + "type": "function", + }, + ], "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_8_FLASH_THOUGHT_SIGNATURE]}, + "reasoning_content": ( + "**My Thought Process on Analyzing the Request and Planning the Next Action**\n\nRight, so my primary " + "objective here is to fulfill the user's request while adhering strictly to the system's operational " + 'guidelines. The user has specifically asked me to "Read the validation file first so we can ' + 'understand how zero and negative quantities are handled." This is a direct instruction that aligns ' + 'perfectly with the overarching system instruction to "Read the relevant files before suggesting a ' + 'fix."\n\nMy internal checklist, based on the system\'s directives, also reminds me to:\n* "Explain ' + 'which file you need next, then call a read-only tool."\n* "Never change files." (This is a ' + 'foundational rule, of course.)\n* "Use the chat_completions client for this project review." (This ' + "context is already established.)\n* Crucially, and directly relevant to the user's query: " + '"Consider zero, negative and non-integer quantities before choosing the next file to ' + 'inspect."\n\nNow, looking at the files I have access to, I see `shop/validation.py` and ' + "`tests/test_validation.py`. The user's request directly points to `shop/validation.py` as the next " + "file to examine for validation logic. This makes sense, as the name itself implies it's where the " + "core validation rules reside.\n\nBefore I jump straight into invoking a tool, the system requires me " + "to articulate *why* I'm selecting this file and *what* I intend to do with it. So, I need to " + "explain that I'm going to inspect `shop/validation.py` specifically to understand its handling of " + "quantities, particularly concerning zero, negative, and non-integer values, as prompted by the user " + "and reinforced by the system's guidance.\n\nFollowing that explanation, I'll then execute the action " + "by calling a read-only tool, `read_file`. I'll specify `shop/validation.py` as the path, and to get " + "a good initial overview, I'll plan to read a reasonable chunk, say from line 1 to around line 50, " + "to capture the initial validation logic. This seems like a good starting point to gather the " + "necessary context for the user's specific inquiry.\n\n" + ), + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "**My Thought Process on Analyzing the Request and Planning the Next Action**\n\nRight, so my primary " + "objective here is to fulfill the user's request while adhering strictly to the system's operational " + 'guidelines. The user has specifically asked me to "Read the validation file first so we can ' + 'understand how zero and negative quantities are handled." This is a direct instruction that aligns ' + 'perfectly with the overarching system instruction to "Read the relevant files before suggesting a ' + 'fix."\n\nMy internal checklist, based on the system\'s directives, also reminds me to:\n* "Explain ' + 'which file you need next, then call a read-only tool."\n* "Never change files." (This is a ' + 'foundational rule, of course.)\n* "Use the chat_completions client for this project review." (This ' + "context is already established.)\n* Crucially, and directly relevant to the user's query: " + '"Consider zero, negative and non-integer quantities before choosing the next file to ' + 'inspect."\n\nNow, looking at the files I have access to, I see `shop/validation.py` and ' + "`tests/test_validation.py`. The user's request directly points to `shop/validation.py` as the next " + "file to examine for validation logic. This makes sense, as the name itself implies it's where the " + "core validation rules reside.\n\nBefore I jump straight into invoking a tool, the system requires me " + "to articulate *why* I'm selecting this file and *what* I intend to do with it. So, I need to " + "explain that I'm going to inspect `shop/validation.py` specifically to understand its handling of " + "quantities, particularly concerning zero, negative, and non-integer values, as prompted by the user " + "and reinforced by the system's guidance.\n\nFollowing that explanation, I'll then execute the action " + "by calling a read-only tool, `read_file`. I'll specify `shop/validation.py` as the path, and to get " + "a good initial overview, I'll plan to read a reasonable chunk, say from line 1 to around line 50, " + "to capture the initial validation logic. This seems like a good starting point to gather the " + "necessary context for the user's specific inquiry.\n\n" + ), + }, + ], + "provider_specific_fields": { + "thought_signatures": [ + ( + "AY89a1/K8QT9La7l3gDc5jImwnXSF0xUuyg8rAUnAyYS4c2/Yjd/fV3Ve1tHSww5bSooGMdZ0oqljadXXxUqYi+7fkKOWD0Ij80K" + "LFu1O//r13u8cIpIqUxEi3lNiEn8rK6WVW5DPTMTBddZBL+/DcpCEi7U3m5vGYfoJUm9AJwQB8XDRdqLdA1hfiRnCqGpLJsFAQ2g" + "Ln3FpQUPR3+FRoWRlJnNRjk8vWaq3mBKHSHttLMRd3rpvo6VAAZbs2OGoBJXFXA9S+RyN532kDK3fZIGGW2Fk+FSD9EJ1hN1+Qeh" + "v8gzcxyZOin2O/ONFWEQ4zf8lHTzaostuIMLkKFM3v/ssK+Y4VgS0sPFr7VFByhczwoDx73oh141iDD5ch7tqAgnR7YikrWQDuV2" + "8FBSB7+MQ3BrBSt6XUvYgcHsGNbg0giE3D3pFDR7uvLfvgjhFFdIqnCWWmPfBgZ7GOD55unG0IUPA2udP65xg88r8TjfpQOUsOnN" + "KZLOiKovzHrXZX2L0tRGha5nE8ICi427Mc5YF9lm0Ilpo/BrGuuusMG3Zx9RxRyjRa9+zWPANRrvuq7SOVnFvAss4uu9PYf5Orwn" + "nGFJ0xjP1hn22yXOxRYy62ySQQ1ValpWXjPPXvGKZRiUArCuvrBvDEZoUfJhlcVXs26kYnqRikRLev4GZGvBIDnGHyWLvMo0gx4r" + "1kBnAGl5WyfZJu5SFywm/o5fMP70zVUoVdY+zC885uqB5pEV+VsAMyy98NumfmBF7kGtIjbt4l+hAqpzM1ME1z0kRqN7n9JEKn0H" + "DlsQAD9V582O2bBVSTRzF8aFJ7RKohGhrFqyGmYLA+ZAv5Z4kJ1HBZg1LEz2G88MT2vznP6lCEH0m9da48Q5/85pNQ5dlGGEkhpP" + "I+xazh6EZ0cFNGdNY1vAQY5rBQ8LPwizNSKenYvqwMdobWy8xReIQWCsOk2l4QCtlu8kC+iabWrsCPOfjdQmYTlpI+emQwssWXYU" + "UQ+wDGl7QWoaSAnwGu+XQMQ9ndaUycPdBu5+cuw815uiNNtIVfeKejFIbLF66Jq/duKfZ9tgWthPzpoM50u9aie2FZwNOuEtaLIv" + "WcdtBpY9LvnAemSuDUTLLSuiQYwBwRq8HKdFARJ362Wf8gg+ldoeXEV9R6+cscPaeY5j/+E/rsTWyxixg614qiFQyF5xL/7DWjUI" + "LBuIrmJUIVcu8LelTUslepTGF6CBTJ0RABv8pK7S8dUDy+HXtvT8bmz8l9zOsaTagC1jUpvaeGpi44gorN9KU7z56FMxoKSeKF7M" + "B2zuTFBN/GekI9gddLfQRaVK/9LAD4j+uBzdt/XICe2kGqjNu3ds9tnx82/9beezRyTRfFQ+p1V/jfaEt8AMTZ5I2IZQAkgkAnXa" + "KieRKs8qKbkF5yjhtI0krsyg4+BDQZzog+dgkxqYxLDuNyd6hN9HlnMEBAwW8hxWGp4WE8GvL2IU529JVMaBvK/NcrGNnPW/m158" + "BwaIFSfbSxQbqjv6KKqMi30i2bowXBAJit2YQHb3VN1reFBynM9mRm71PGLAbizdVS5VIqNI5zGquZrgpmTqqMF/AjsWpVSTo6Gi" + "WD3O2PJWReehkH3aMJMFeu3jv0/Ru+Ihw1zwFcZr0Yh7rK2NEYQyZjcmf5/vX0rBh89HEVdc78j16arXcFWHNZS2lLAm0NC+Xmak" + "WIP3YUcba6POPYKwqP2huUccyyGg+qFPTcWF488PNm4H86YQRrnUSkPEofjMuaLMrp3vS1TgpjuUZRiZ" + ), + ], + }, }, "provider_specific_fields": {"native_finish_reason": "STOP"}, - } + }, ], "usage": { - "completion_tokens": 109, - "prompt_tokens": 9, - "total_tokens": 118, - "completion_tokens_details": {"reasoning_tokens": 107, "text_tokens": 2}, - "prompt_tokens_details": {"text_tokens": 9}, + "completion_tokens": 396, + "prompt_tokens": 246, + "total_tokens": 642, + "completion_tokens_details": {"reasoning_tokens": 331, "text_tokens": 65}, + "prompt_tokens_details": {"text_tokens": 246}, }, "vertex_ai_grounding_metadata": [], "vertex_ai_url_context_metadata": [], @@ -941,74 +976,334 @@ GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( }, ) + GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "vertex_ai/gemini-3.1-pro-preview", - "max_tokens": 1024, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "vertex_ai/gemini-3.1-pro-preview"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.1-pro-preview:generateContent", expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, + expected_provider_request=VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, "candidates": [ { "content": { "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], + "parts": [ + { + "text": ( + "**My Thought Process**\n\nAlright, first things first, I need to get a handle on how this system is " + "dealing with product quantities, specifically those tricky zero and negative values. The best place " + "to start is the validation logic itself. I'll bet that `shop/validation.py` file holds the key to " + "this.\n\nTo get to the bottom of it, I'm going to use the `read_file` tool. I'll point it directly at " + "`shop/validation.py`. I don't need the whole file, just enough to see the relevant validation " + "functions. I'll set a reasonable range, say from line 1 to line 50. This should be more than enough " + "to capture the `validate_quantity` function, which I anticipate will be at or around line 18, and " + "give me the insights I'm looking for into how zero and negative quantities are being treated.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I'll read the validation file to see the implementation details for the quantity validation, " + "specifically focusing on how zero and negative values are processed." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"end_line": 50, "start_line": 1, "path": "shop/validation.py"}, + "id": "call_3113284", + }, + "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_CHAT_COMPLETIONS_THOUGHT_SIGNATURE_0, + }, + ], }, "finishReason": "STOP", - } + }, ], "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 1, - "totalTokenCount": 99, + "promptTokenCount": 246, + "candidatesTokenCount": 61, + "totalTokenCount": 461, "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 1}], - "thoughtsTokenCount": 89, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 246}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 61}], + "thoughtsTokenCount": 154, }, "modelVersion": "gemini-3.1-pro-preview", - "createTime": "2026-10-05T22:55:14.874538Z", - "responseId": "0irEaqqwNbCdq8YP3L2mqQ4", + "createTime": "2026-10-15T01:13:17.173080Z", + "responseId": "rSjQapjICtCX4_UPjurAmAo", }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": "0irEaqqwNbCdq8YP3L2mqQ4", - "created": ANY, + **CHAT_COMPLETIONS_RESPONSE, "model": "vertex_ai/gemini-3.1-pro-preview", - "object": "chat.completion", "choices": [ { - "finish_reason": "stop", + "finish_reason": "tool_calls", "index": 0, "message": { - "content": "Hello.", + "content": ( + "I'll read the validation file to see the implementation details for the quantity validation, " + "specifically focusing on how zero and negative values are processed." + ), "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": ( + "AY89a18NmP/rpOzYqPT2gXtTGPJwCXopurRCh2SERSxHAbUcM+kBs62qrKZf6X3eHEdd/leFa2/wBviqJ6WkkEdIsgz94SdkfwWm" + "53Pd6IC20z+9KTq6PpsXs5CJQzMRyWvSBBolC4JHE99knIqr4nJbUlXHYCknKnExuUNUuWzkcY2zsO/uX81og8IwGDikReqErvvU" + "+EVyAV0TSDb3Pg0A8eC7RCHBLPfT0ljwzNJDibRUkwoxdKcPkMvdbYhZEAnkOPD7N/CpITooxxwBYVArsbDW1oA4kL78A2gnbsNi" + "Y2esZSaKgxrftJk1H06rW5b7tAk7y4wMjbj7GD9XQJJ3HzLdUznXNvLRZjkNcr8evFgKPq3YIPqpuCn5klH09/qjGNZRkKoYriBS" + "f95iWKvGZ7jwEHu+45Qw46o0lID1TFZsrnS4WiwllgqAhvvFZ//DbCtpjMIztpTGYD754oc60k50p6Ny14oPDBlNMOVqwiFGq9QC" + "FQ7Vzilx+rqAoVdOzvyGSf6+SfXOT0Jv/raw+HIIA2bhOqurVqgdS4CoxdpCcZ5CmOgN4CIA9rQcuxZ/dVaT2w/UL0fUJOAN3kSU" + "ngcI+FcqIjlbF+IbIEGphpBhsIaTd3qXVCH/57K0tP7lVu9GoAU9zNu/6YPIWZnE9ABUBfQkyTR/DdFZm3Hfsjt7m5YkPlm+kJNH" + "iHHwdjTo+liy9gBFnQbl+mFgv3e0bcdy+a0pa6mUyluy0IWrKFBVFcfEep4vD7+CRTpkpwhwzL6+v2t3LWGcJGZV6GS/qbbGR3ys" + "Cyr70DrZtaEnbOnLN3UYP6zDcDfKYTn7ub3hIjaME9LNjg==" + ), + }, + "function": { + "arguments": '{"end_line": 50, "start_line": 1, "path": "shop/validation.py"}', + "name": "read_file", + }, + "id": ( + "call_3113284__thought__AY89a18NmP/rpOzYqPT2gXtTGPJwCXopurRCh2SERSxHAbUcM+kBs62qrKZf6X3eHEdd/leFa2/wB" + "viqJ6WkkEdIsgz94SdkfwWm53Pd6IC20z+9KTq6PpsXs5CJQzMRyWvSBBolC4JHE99knIqr4nJbUlXHYCknKnExuUNUuWzkcY2zs" + "O/uX81og8IwGDikReqErvvU+EVyAV0TSDb3Pg0A8eC7RCHBLPfT0ljwzNJDibRUkwoxdKcPkMvdbYhZEAnkOPD7N/CpITooxxwBY" + "VArsbDW1oA4kL78A2gnbsNiY2esZSaKgxrftJk1H06rW5b7tAk7y4wMjbj7GD9XQJJ3HzLdUznXNvLRZjkNcr8evFgKPq3YIPqpu" + "Cn5klH09/qjGNZRkKoYriBSf95iWKvGZ7jwEHu+45Qw46o0lID1TFZsrnS4WiwllgqAhvvFZ//DbCtpjMIztpTGYD754oc60k50p" + "6Ny14oPDBlNMOVqwiFGq9QCFQ7Vzilx+rqAoVdOzvyGSf6+SfXOT0Jv/raw+HIIA2bhOqurVqgdS4CoxdpCcZ5CmOgN4CIA9rQcu" + "xZ/dVaT2w/UL0fUJOAN3kSUngcI+FcqIjlbF+IbIEGphpBhsIaTd3qXVCH/57K0tP7lVu9GoAU9zNu/6YPIWZnE9ABUBfQkyTR/D" + "dFZm3Hfsjt7m5YkPlm+kJNHiHHwdjTo+liy9gBFnQbl+mFgv3e0bcdy+a0pa6mUyluy0IWrKFBVFcfEep4vD7+CRTpkpwhwzL6+v" + "2t3LWGcJGZV6GS/qbbGR3ysCyr70DrZtaEnbOnLN3UYP6zDcDfKYTn7ub3hIjaME9LNjg==" + ), + "type": "function", + }, + ], "images": [], - "thinking_blocks": [], - "provider_specific_fields": {"thought_signatures": [GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE]}, + "reasoning_content": ( + "**My Thought Process**\n\nAlright, first things first, I need to get a handle on how this system is " + "dealing with product quantities, specifically those tricky zero and negative values. The best place " + "to start is the validation logic itself. I'll bet that `shop/validation.py` file holds the key to " + "this.\n\nTo get to the bottom of it, I'm going to use the `read_file` tool. I'll point it directly at " + "`shop/validation.py`. I don't need the whole file, just enough to see the relevant validation " + "functions. I'll set a reasonable range, say from line 1 to line 50. This should be more than enough " + "to capture the `validate_quantity` function, which I anticipate will be at or around line 18, and " + "give me the insights I'm looking for into how zero and negative quantities are being treated.\n\n" + ), + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "**My Thought Process**\n\nAlright, first things first, I need to get a handle on how this system is " + "dealing with product quantities, specifically those tricky zero and negative values. The best place " + "to start is the validation logic itself. I'll bet that `shop/validation.py` file holds the key to " + "this.\n\nTo get to the bottom of it, I'm going to use the `read_file` tool. I'll point it directly at " + "`shop/validation.py`. I don't need the whole file, just enough to see the relevant validation " + "functions. I'll set a reasonable range, say from line 1 to line 50. This should be more than enough " + "to capture the `validate_quantity` function, which I anticipate will be at or around line 18, and " + "give me the insights I'm looking for into how zero and negative quantities are being treated.\n\n" + ), + }, + ], + "provider_specific_fields": { + "thought_signatures": [ + ( + "AY89a18NmP/rpOzYqPT2gXtTGPJwCXopurRCh2SERSxHAbUcM+kBs62qrKZf6X3eHEdd/leFa2/wBviqJ6WkkEdIsgz94SdkfwWm" + "53Pd6IC20z+9KTq6PpsXs5CJQzMRyWvSBBolC4JHE99knIqr4nJbUlXHYCknKnExuUNUuWzkcY2zsO/uX81og8IwGDikReqErvvU" + "+EVyAV0TSDb3Pg0A8eC7RCHBLPfT0ljwzNJDibRUkwoxdKcPkMvdbYhZEAnkOPD7N/CpITooxxwBYVArsbDW1oA4kL78A2gnbsNi" + "Y2esZSaKgxrftJk1H06rW5b7tAk7y4wMjbj7GD9XQJJ3HzLdUznXNvLRZjkNcr8evFgKPq3YIPqpuCn5klH09/qjGNZRkKoYriBS" + "f95iWKvGZ7jwEHu+45Qw46o0lID1TFZsrnS4WiwllgqAhvvFZ//DbCtpjMIztpTGYD754oc60k50p6Ny14oPDBlNMOVqwiFGq9QC" + "FQ7Vzilx+rqAoVdOzvyGSf6+SfXOT0Jv/raw+HIIA2bhOqurVqgdS4CoxdpCcZ5CmOgN4CIA9rQcuxZ/dVaT2w/UL0fUJOAN3kSU" + "ngcI+FcqIjlbF+IbIEGphpBhsIaTd3qXVCH/57K0tP7lVu9GoAU9zNu/6YPIWZnE9ABUBfQkyTR/DdFZm3Hfsjt7m5YkPlm+kJNH" + "iHHwdjTo+liy9gBFnQbl+mFgv3e0bcdy+a0pa6mUyluy0IWrKFBVFcfEep4vD7+CRTpkpwhwzL6+v2t3LWGcJGZV6GS/qbbGR3ys" + "Cyr70DrZtaEnbOnLN3UYP6zDcDfKYTn7ub3hIjaME9LNjg==" + ), + ], + }, }, "provider_specific_fields": {"native_finish_reason": "STOP"}, - } + }, ], "usage": { - "completion_tokens": 90, - "prompt_tokens": 9, - "total_tokens": 99, - "completion_tokens_details": {"reasoning_tokens": 89, "text_tokens": 1}, - "prompt_tokens_details": {"text_tokens": 9}, + "completion_tokens": 215, + "prompt_tokens": 246, + "total_tokens": 461, + "completion_tokens_details": {"reasoning_tokens": 154, "text_tokens": 61}, + "prompt_tokens_details": {"text_tokens": 246}, + }, + "vertex_ai_grounding_metadata": [], + "vertex_ai_url_context_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_citation_metadata": [], + }, +) + + +GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "vertex_ai/gemini-4-argon"}, + expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-4-argon:generateContent", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, + expected_provider_request=VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": ( + "**My Thought Process on Reviewing Shop Logic**\n\nAlright, my first order of business is to get a " + "firm grasp on how the shop's validation logic handles edge cases, specifically concerning " + "quantities. The prompt explicitly mentions zero, negative, and non-integer quantities, and I know " + "from my experience that these are often the sources of bugs. To understand this thoroughly, I need " + "to dive into the `shop/validation.py` file. This file is the most likely place where these crucial " + "validation rules are defined. Once I've reviewed that, I can proceed with a clearer picture of how " + "the system is designed to handle these potentially problematic inputs. Therefore, my next action is " + "to request a read of `shop/validation.py` so I can examine its contents.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I need to read `shop/validation.py` next to see how zero, negative, and non-integer quantities are " + "currently validated." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"end_line": 50, "start_line": 1, "path": "shop/validation.py"}, + "id": "call_290084", + }, + "thoughtSignature": GEMINI_4_ARGON_CHAT_COMPLETIONS_THOUGHT_SIGNATURE_0, + }, + ], + }, + "finishReason": "STOP", + }, + ], + "usageMetadata": { + "promptTokenCount": 414, + "candidatesTokenCount": 67, + "totalTokenCount": 624, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 414}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 67}], + "thoughtsTokenCount": 143, + }, + "modelVersion": "gemini-4-argon", + "createTime": "2026-10-15T01:13:27.878297Z", + "responseId": "tyjQatnNNZyQ4_UPpcr16QM", + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "vertex_ai/gemini-4-argon", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": ( + "I need to read `shop/validation.py` next to see how zero, negative, and non-integer quantities are " + "currently validated." + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": ( + "AY89a19UcUynGbcdZOu5ucJyuVTEBkP7BWc+WQRD+zdel8qY2zqNh1OLS6DejBKWK9xndkK8coxMqPRuuQytzyB+zVvJz0ouSNh0" + "nH3Bm0p+S9hFJDzNhlHrk4PRtU0OWxv5SKWtzQURmeBDk5mJV/MU9yJSDs7g+csm6juZi2Lang76vPWyQOHTVPYnJVzLHkP73H0D" + "WnVPXSdNzBgqPzopqf3GoP+EvALphWyi0Zp/1uv7m4TKbx9dRZjlO15LaqpksnkDvwkF6o8KfaYrC/Skh+xYxlk40yxiiaYv5sr4" + "PbGutzCGNWCZxjbBCbQBPGMFIL3fDGwLavbGd71tWKcOaSBr3oYbsw5HsfWMFHV8+TQ7rou77MCMWaiJ3mulflSmdBkkTq0m1bPm" + "iIbwYvVPs6CCtTTZgGYTZhRefefwD1Dx4Y5epmGy1+jST7zikArzdZeexaCTTrIWO4rvqzPdAS65XHnfXjgYEPYK0YUnX6StK/ln" + "Lrwdb+AK6j+Q2eLuKpx/HJE0QDZ9BgXpw/CQ6my0s2rI3Tjlbun8Bdl7MyFuCbstbTA+uu1RXXv2M1RThJQHiuDGvgzCNg/sgyCb" + "zluzK/vp2nzki+ESsFeForBBSSCLVAep2tiqKhIOWPWBHYjH04qfgyWkylbRbZeA32+BPYViEcZa+Imi+aYXu241RDOSWRMJvbsJ" + "oDY15ThjQ+sgLNKJkBoravc7+sQ1Abt+jwCcO9zHBquTWn8G/X0IggqHNuDnbWEW3S9HdxPXw7ZV0ZYXXkZXIYwFIt6Ji7LpXlEY" + "X0hOwrBdq2ufKAv+EWBQyCU3oyBrIGeVdVx25BiLFsVJzqw3xLTUgfelHyawgwtGvd7J+mQBPp4YH1Rk6fbQMwDEWipSiltG+jG9" + "ZqXUiF1NtN30tw4a4h2tA03uBV1yXhv6+fL7g2zmN6c4GVX9IzAS60UUnXIMlBgCXjxCU591TuPufe6mJQ==" + ), + }, + "function": { + "arguments": '{"end_line": 50, "start_line": 1, "path": "shop/validation.py"}', + "name": "read_file", + }, + "id": ( + "call_290084__thought__AY89a19UcUynGbcdZOu5ucJyuVTEBkP7BWc+WQRD+zdel8qY2zqNh1OLS6DejBKWK9xndkK8coxMqP" + "RuuQytzyB+zVvJz0ouSNh0nH3Bm0p+S9hFJDzNhlHrk4PRtU0OWxv5SKWtzQURmeBDk5mJV/MU9yJSDs7g+csm6juZi2Lang76vP" + "WyQOHTVPYnJVzLHkP73H0DWnVPXSdNzBgqPzopqf3GoP+EvALphWyi0Zp/1uv7m4TKbx9dRZjlO15LaqpksnkDvwkF6o8KfaYrC/" + "Skh+xYxlk40yxiiaYv5sr4PbGutzCGNWCZxjbBCbQBPGMFIL3fDGwLavbGd71tWKcOaSBr3oYbsw5HsfWMFHV8+TQ7rou77MCMWa" + "iJ3mulflSmdBkkTq0m1bPmiIbwYvVPs6CCtTTZgGYTZhRefefwD1Dx4Y5epmGy1+jST7zikArzdZeexaCTTrIWO4rvqzPdAS65XH" + "nfXjgYEPYK0YUnX6StK/lnLrwdb+AK6j+Q2eLuKpx/HJE0QDZ9BgXpw/CQ6my0s2rI3Tjlbun8Bdl7MyFuCbstbTA+uu1RXXv2M1" + "RThJQHiuDGvgzCNg/sgyCbzluzK/vp2nzki+ESsFeForBBSSCLVAep2tiqKhIOWPWBHYjH04qfgyWkylbRbZeA32+BPYViEcZa+I" + "mi+aYXu241RDOSWRMJvbsJoDY15ThjQ+sgLNKJkBoravc7+sQ1Abt+jwCcO9zHBquTWn8G/X0IggqHNuDnbWEW3S9HdxPXw7ZV0Z" + "YXXkZXIYwFIt6Ji7LpXlEYX0hOwrBdq2ufKAv+EWBQyCU3oyBrIGeVdVx25BiLFsVJzqw3xLTUgfelHyawgwtGvd7J+mQBPp4YH1" + "Rk6fbQMwDEWipSiltG+jG9ZqXUiF1NtN30tw4a4h2tA03uBV1yXhv6+fL7g2zmN6c4GVX9IzAS60UUnXIMlBgCXjxCU591TuPufe" + "6mJQ==" + ), + "type": "function", + }, + ], + "images": [], + "reasoning_content": ( + "**My Thought Process on Reviewing Shop Logic**\n\nAlright, my first order of business is to get a " + "firm grasp on how the shop's validation logic handles edge cases, specifically concerning " + "quantities. The prompt explicitly mentions zero, negative, and non-integer quantities, and I know " + "from my experience that these are often the sources of bugs. To understand this thoroughly, I need " + "to dive into the `shop/validation.py` file. This file is the most likely place where these crucial " + "validation rules are defined. Once I've reviewed that, I can proceed with a clearer picture of how " + "the system is designed to handle these potentially problematic inputs. Therefore, my next action is " + "to request a read of `shop/validation.py` so I can examine its contents.\n\n" + ), + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "**My Thought Process on Reviewing Shop Logic**\n\nAlright, my first order of business is to get a " + "firm grasp on how the shop's validation logic handles edge cases, specifically concerning " + "quantities. The prompt explicitly mentions zero, negative, and non-integer quantities, and I know " + "from my experience that these are often the sources of bugs. To understand this thoroughly, I need " + "to dive into the `shop/validation.py` file. This file is the most likely place where these crucial " + "validation rules are defined. Once I've reviewed that, I can proceed with a clearer picture of how " + "the system is designed to handle these potentially problematic inputs. Therefore, my next action is " + "to request a read of `shop/validation.py` so I can examine its contents.\n\n" + ), + }, + ], + "provider_specific_fields": { + "thought_signatures": [ + ( + "AY89a19UcUynGbcdZOu5ucJyuVTEBkP7BWc+WQRD+zdel8qY2zqNh1OLS6DejBKWK9xndkK8coxMqPRuuQytzyB+zVvJz0ouSNh0" + "nH3Bm0p+S9hFJDzNhlHrk4PRtU0OWxv5SKWtzQURmeBDk5mJV/MU9yJSDs7g+csm6juZi2Lang76vPWyQOHTVPYnJVzLHkP73H0D" + "WnVPXSdNzBgqPzopqf3GoP+EvALphWyi0Zp/1uv7m4TKbx9dRZjlO15LaqpksnkDvwkF6o8KfaYrC/Skh+xYxlk40yxiiaYv5sr4" + "PbGutzCGNWCZxjbBCbQBPGMFIL3fDGwLavbGd71tWKcOaSBr3oYbsw5HsfWMFHV8+TQ7rou77MCMWaiJ3mulflSmdBkkTq0m1bPm" + "iIbwYvVPs6CCtTTZgGYTZhRefefwD1Dx4Y5epmGy1+jST7zikArzdZeexaCTTrIWO4rvqzPdAS65XHnfXjgYEPYK0YUnX6StK/ln" + "Lrwdb+AK6j+Q2eLuKpx/HJE0QDZ9BgXpw/CQ6my0s2rI3Tjlbun8Bdl7MyFuCbstbTA+uu1RXXv2M1RThJQHiuDGvgzCNg/sgyCb" + "zluzK/vp2nzki+ESsFeForBBSSCLVAep2tiqKhIOWPWBHYjH04qfgyWkylbRbZeA32+BPYViEcZa+Imi+aYXu241RDOSWRMJvbsJ" + "oDY15ThjQ+sgLNKJkBoravc7+sQ1Abt+jwCcO9zHBquTWn8G/X0IggqHNuDnbWEW3S9HdxPXw7ZV0ZYXXkZXIYwFIt6Ji7LpXlEY" + "X0hOwrBdq2ufKAv+EWBQyCU3oyBrIGeVdVx25BiLFsVJzqw3xLTUgfelHyawgwtGvd7J+mQBPp4YH1Rk6fbQMwDEWipSiltG+jG9" + "ZqXUiF1NtN30tw4a4h2tA03uBV1yXhv6+fL7g2zmN6c4GVX9IzAS60UUnXIMlBgCXjxCU591TuPufe6mJQ==" + ), + ], + }, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + }, + ], + "usage": { + "completion_tokens": 210, + "prompt_tokens": 414, + "total_tokens": 624, + "completion_tokens_details": {"reasoning_tokens": 143, "text_tokens": 67}, + "prompt_tokens_details": {"text_tokens": 414}, }, "vertex_ai_grounding_metadata": [], "vertex_ai_url_context_metadata": [], diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_anthropic.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_anthropic.py index febbd659443..ac13d5d25df 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_anthropic.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_anthropic.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py index 635d75571b2..6c588339eab 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py @@ -6,6 +6,7 @@ from integration.translation.chat_completions.bases.azure import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -22,6 +23,7 @@ from integration.translation.runner import assert_translation GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure_ai.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure_ai.py index f3accebae1e..f65a78cc9e4 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure_ai.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure_ai.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py index 69a1efd7825..88f1fe2d823 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_converse.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_invoke.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_invoke.py index 9f3fc4e4574..a135a54c9f6 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_invoke.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_invoke.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_mantle.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_mantle.py index 32ecec11b15..9e6652b100f 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_mantle.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_bedrock_mantle.py @@ -9,7 +9,9 @@ from integration.translation.chat_completions.bases.bedrock_mantle import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, + GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, ) from integration.translation.runner import assert_translation @@ -26,6 +28,8 @@ from integration.translation.runner import assert_translation GPT_5_6_LUNA_TEST_CASE, GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, + GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_gemini.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_gemini.py index f989bb50753..057a0044bae 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_gemini.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_gemini.py @@ -6,13 +6,19 @@ from integration.translation.chat_completions.bases.gemini import ( GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ) from integration.translation.runner import assert_translation @pytest.mark.parametrize( "case", - [GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE], + [ + GEMINI_3_5_FLASH_TEST_CASE, + GEMINI_3_8_FLASH_TEST_CASE, + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, + ], ids=lambda case: case.id, ) def test_chat_completions_basic_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py index d73155f218e..b6a60721b65 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py @@ -6,6 +6,7 @@ from integration.translation.chat_completions.bases.openai import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -22,6 +23,7 @@ from integration.translation.runner import assert_translation GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_vertex_ai.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_vertex_ai.py index e9d28a0121d..71cbd84543e 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_vertex_ai.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_vertex_ai.py @@ -1,19 +1,19 @@ import pytest - from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase from integration.translation.chat_completions.bases.vertex_ai import ( CLAUDE_FABLE_5_1_TEST_CASE, CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_SONNET_4_6_TEST_CASE, - CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, - CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, - GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ) from integration.translation.runner import assert_translation @@ -31,6 +31,7 @@ from integration.translation.runner import assert_translation GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_azure_ai.py b/tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_azure_ai.py new file mode 100644 index 00000000000..467b8137e85 --- /dev/null +++ b/tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_azure_ai.py @@ -0,0 +1,215 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.bases.azure_ai import ( + CLAUDE_FABLE_5_1_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, +) +from integration.translation.runner import assert_translation + +CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0: Final = ( + "CAQS3gYKEAgSGAI4AUIIdGhpbmtpbmcSDEoUbRA8/sNqRoVoJRoMjpP0glsrmS7QMeraIjDxIxhKjyeY5Y65IZo5YTWEVqymcjH8k7a0mAjW" + "bQoNTG9gw0R+WqIckDwWmnUrlFgq+wUl+/iw8hZ5IAryY71AHQuvdYaGuEgmN8jqzGujRvAeW4tw3zLG0td8x1lyoJk02iTU5aQV0wO5uO87" + "wJqAEJ6bbheWT9m9l+M3tW9DOFsQMHYpj95wz6crrlZxzhBgMN7dZMzy+l+ATwMSxpQHX4FXRNAsz63J2tOADTeHSFXeP6Evn5abAhSiznHf" + "ezSB2bRv6eBRVi7hDR2MgVJ1CEo5b2YtGroJPzVJxZRZpyHKg04dRkAUn0uQb+N5FxJ5I4v2phqqMLqpvDR8sU14Q91v7SstrT7NyRlAFhv9" + "pzocddLhJkyHJpFctQr0D1k1oiyPCWpWdRVMEPrAybvkh6vVij2l6CP/kBfD8JUUulRCPO+xoyfAglQJsA4zc1gRCggBKbim+26oazBYat8K" + "+1RO5iRzQe/gvs8DwQSPML/YWqzGK/qd1G7yJujaMnkBt9Jew44DQ/eUgKP5030zKd/ft+jmj8uu2PNP6zZrvZA5grdTSY6pSnu4X++1wE5r" + "NExLgC/TJgDejVzCazAP9MsDnXz7nv+6yVytEsL//xOVq7s5nF0IYpNdawt1bAzNaHAnIMH8meeksDnnRJuBKt4mhg0nPgueOXgo7RLEUHWm" + "MYakXV5wFNLY1oAUV4RpQJ+zF+/mJ/f7AHmghcxoPnPPPlISar3f9w00CQRVQjJDkm8ozxWyLN6AXhDQfQEDvYWPl33VaOcv7TpLAJ7HkWyk" + "AJA8oB3LflgBJCSOXL67LGiGeSw8eR0bp89+Yux2C9MzORRvE8asjfl1W6NFWJzsvpNYcRYm0QREuNDRabg+mkcfwpDJyeaopSsNZLOMS7zq" + "mwFMPrmW9CY+MJTxG0pQX1G8Ln59hhPu242z8l6wP1eqBbRzV54RMjE0wBGV7vfoPkqBzxGAj0Cr+/BeNbM4ne9k4pL9s5JYRsWhhhF1mkN7" + "pFlOxz8cswlKkAaq/KnCWp2ptVgzxJazuokShk4hnnbwcdxaDVX2NynBGnmbcpfG8zso2GC52Ke+GAE=" +) + +CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_HAIKU_5_5_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_HAIKU_5_5_TEST_CASE.mock_provider_response, + "id": "msg_011Cg37bzPpF6nmTd4cxR6zH", + "content": [ + { + "type": "thinking", + "thinking": "I'll start by reading shop/validation.py to understand its contents, focusing on the " + "first portion of the file before deciding if more context is needed.\n\n", + "signature": CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` around the `validate_quantity` definition at line 18 " + "to see how it treats zero, negative, and non-integer values.", + }, + { + "type": "tool_use", + "id": "toolu_01L2rdexX7pthHtxd9WR8UbE", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 252, + "output_tokens_details": {"thinking_tokens": 103}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **CLAUDE_HAIKU_5_5_TEST_CASE.expected_litellm_response, + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll read `shop/validation.py` around the `validate_quantity` definition at line 18 " + "to see how it treats zero, negative, and non-integer values.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_01L2rdexX7pthHtxd9WR8UbE", + "type": "function", + }, + ], + "reasoning_content": "I'll start by reading shop/validation.py to understand its contents, focusing on the " + "first portion of the file before deciding if more context is needed.\n\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll start by reading shop/validation.py to understand its contents, focusing on the " + "first portion of the file before deciding if more context is needed.\n\n", + "signature": CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll start by reading shop/validation.py to understand its contents, focusing on the " + "first portion of the file before deciding if more context is needed.\n\n", + "signature": CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + ], + }, + }, + }, + ], + "usage": { + "completion_tokens": 252, + "prompt_tokens": 880, + "total_tokens": 1132, + "completion_tokens_details": {"reasoning_tokens": 103, "text_tokens": 149}, + "prompt_tokens_details": { + "cached_tokens": 876, + "text_tokens": 4, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "inference_geo": "global", + "service_tier": "standard", + }, + }, +) + +CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_FABLE_5_1_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_FABLE_5_1_TEST_CASE.mock_provider_response, + "id": "msg_011Cg37V6cvTmK19kQhSJPvQ", + "content": [ + { + "type": "text", + "text": "I'll read the validation file to see how zero and negative quantities are handled.", + }, + { + "type": "tool_use", + "id": "toolu_01Gk8RXusGMKa2tw97hQjuzk", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 123, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "not_available", + }, + }, + expected_litellm_response={ + **CLAUDE_FABLE_5_1_TEST_CASE.expected_litellm_response, + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll read the validation file to see how zero and negative quantities are handled.", + "role": "assistant", + "tool_calls": [ + { + "index": 1, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "name": "read_file", + }, + "id": "toolu_01Gk8RXusGMKa2tw97hQjuzk", + "type": "function", + }, + ], + "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + }, + }, + ], + "usage": { + "completion_tokens": 123, + "prompt_tokens": 880, + "total_tokens": 1003, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 123}, + "prompt_tokens_details": { + "cached_tokens": 876, + "text_tokens": 4, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "inference_geo": "not_available", + "service_tier": "standard", + }, + }, +) + + +@pytest.mark.parametrize( + "case", + [CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE, CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE], + ids=[CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE.id, CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE.id], +) +def test_chat_completions_prompt_cache_hit_azure_ai( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py b/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py index 1707fc71238..4de06c08a75 100644 --- a/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py +++ b/tests/integration/translation/chat_completions/sampling/test_chat_completions_sampling_gemini.py @@ -9,7 +9,11 @@ from integration.translation.chat_completions.bases.gemini import GEMINI_3_8_FLA from integration.translation.chat_completions.bases.vertex_ai import ( GEMINI_3_8_FLASH_TEST_CASE as VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE, ) +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_RESPONSE from integration.translation.runner import assert_translation +from integration.translation.templates.vertex_ai import ( + VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, +) GEMINI_GENERATION_CONFIG: Final = object_value(GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request["generationConfig"]) VERTEX_AI_GENERATION_CONFIG: Final = object_value( @@ -49,6 +53,29 @@ GEMINI_3_8_FLASH_REASONING_EFFORT_TEST_CASE: Final = replace( }, ) +VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0: Final = ( + "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" + "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" + "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" + "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" + "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" + "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" + "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" + "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" + "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" + "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" + "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" + "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" + "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" + "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" + "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" + "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" + "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" + "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" + "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" +) + + VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE, scenario="temperature", @@ -57,6 +84,204 @@ VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_TEST_CASE: Final = replace( **VERTEX_AI_GEMINI_3_8_FLASH_TEST_CASE.expected_provider_request, "generationConfig": {**VERTEX_AI_GENERATION_CONFIG, "temperature": 0.4}, }, + mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": ( + "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " + "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " + "understanding the existing validation logic. This is crucial before I can even begin to think about " + 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' + "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " + "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " + "how different types of quantities are processed and validated. My plan is to read a reasonable " + "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " + "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " + "need to investigate next, keeping in mind the specific constraints of zero, negative, and " + 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' + "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" + ), + "thought": True, + }, + { + "text": ( + "To examine how zero, negative, and non-integer quantities are handled, I need to inspect " + "`shop/validation.py` next." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"start_line": 1, "end_line": 50, "path": "shop/validation.py"}, + "id": "call_1464376", + }, + "thoughtSignature": VERTEX_AI_GEMINI_3_8_FLASH_TEMPERATURE_THOUGHT_SIGNATURE_0, + }, + ], + }, + "finishReason": "STOP", + }, + ], + "usageMetadata": { + "promptTokenCount": 246, + "candidatesTokenCount": 61, + "totalTokenCount": 646, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 246}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 61}], + "thoughtsTokenCount": 339, + }, + "modelVersion": "gemini-3.8-flash", + "createTime": "2026-10-15T01:22:48.202504Z", + "responseId": "6CrQaoiuDNKZq8YPjZCokAw", + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "vertex_ai/gemini-3.8-flash", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": ( + "To examine how zero, negative, and non-integer quantities are handled, I need to inspect " + "`shop/validation.py` next." + ), + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": ( + "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" + "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" + "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" + "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" + "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" + "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" + "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" + "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" + "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" + "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" + "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" + "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" + "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" + "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" + "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" + "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" + "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" + "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" + "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" + ), + }, + "function": { + "arguments": '{"start_line": 1, "end_line": 50, "path": "shop/validation.py"}', + "name": "read_file", + }, + "id": ( + "call_1464376__thought__AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38B" + "NeldTuj8NebwQ7BneePvMzchDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4K" + "nPKk6MXh0at/4JacXcOFRRXDnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qg" + "XUr/qzNSZoFGzRN6m7OzSuTyQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2q" + "Ru6NCqiF+EnTFMUg7g+1cN1nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fy" + "O8SVW6AUe0/S/o0/e//wn6AaFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhP" + "h7S+5lMEvuD8YLITzrC81Xcb4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrU" + "EDZRPAgomcOLBSdK6gQ9pn5z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0Dy" + "UAZM8H2ymbusxnKdIHanI9ZUS/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW" + "1ZetNWVtQl2s+oZXCHOtVJD2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqz" + "TdXmXFLj/bePo7YzfYpLOLlhEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+" + "wN2K6hfqgtJy80rl5ceUT7f6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWG" + "otbx0FiYEDBbi2b7t7L/w3fP3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82Ab" + "XU5sKEKFShEGnJSw9B0OlWuPa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yL" + "yf/Gg9h4ebosT6NVh7AkzS2riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3" + "ZadKxulNV9kkXN2Gc3v/VyBruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKA" + "R15mprXot6VKXkRHfyQJJz0lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUEC" + "EB3gGoIFo9rmHTURX4cAALaoZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7" + "+Pa2VwmCgWUoFFHRbO8ryoyW57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" + ), + "type": "function", + }, + ], + "images": [], + "reasoning_content": ( + "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " + "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " + "understanding the existing validation logic. This is crucial before I can even begin to think about " + 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' + "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " + "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " + "how different types of quantities are processed and validated. My plan is to read a reasonable " + "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " + "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " + "need to investigate next, keeping in mind the specific constraints of zero, negative, and " + 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' + "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" + ), + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "**My Thought Process on Handling Quantity Validation**\n\nAlright, to properly address the handling " + "of zero, negative, and potentially non-integer quantities, my immediate next step needs to be " + "understanding the existing validation logic. This is crucial before I can even begin to think about " + 'suggesting any fixes or further code. My protocol dictates that I must "read the relevant files ' + "before suggesting a fix,\" and that's exactly what I'm prioritizing.\n\nTherefore, I need to examine " + "the `shop/validation.py` file. This is where I anticipate finding the rules and checks that govern " + "how different types of quantities are processed and validated. My plan is to read a reasonable " + "initial segment of this file, say lines 1 through 50, to get a solid grasp of the current " + "implementation. After that, I'll be able to determine precisely *which* other files, if any, I'll " + "need to investigate next, keeping in mind the specific constraints of zero, negative, and " + 'non-integer values.\n\nTo do this, I will now use the `read_file` tool:\n\n```json\n{\n "tool_code": ' + "\"print(read_file(path='shop/validation.py', start_line=1, end_line=50))\"\n}\n```\n\n" + ), + }, + ], + "provider_specific_fields": { + "thought_signatures": [ + ( + "AY89a18+3vC8Y6wtLQ8YLNoWj0pZ1qcMTAdhuIp6v+o7VjO8nN81l72KF3fiSKtgKQYvOKc4hz38BNeldTuj8NebwQ7BneePvMzc" + "hDx/5j5DXPVIIW7bRE2vYMhffaNpyjdltlQRDAxhiqjbSHIIGZQdxxAE7GfdGxlhstemN91VzQR4KnPKk6MXh0at/4JacXcOFRRX" + "DnYmUqj1a1tSIoSjeTsHMvSj5OpJyxxa8Fafd/yqwL8g4scNZgGzckn4O5quaQmJT9VPtoAsmh8qgXUr/qzNSZoFGzRN6m7OzSuT" + "yQ246hxKwp0+ZB9P+Nn5fvG96JVXsRq7dF00HBAbhebjUfbNTmIQauZHGSGbF2e+ck0aG5Lf8se2qRu6NCqiF+EnTFMUg7g+1cN1" + "nIXLw7X6cAPPrpXvVdkfAoHksgSFWY5uUYwW9yDTZYwOWqF6bxDapPFU5HmJj/Bf5YuVH7VhGB7fyO8SVW6AUe0/S/o0/e//wn6A" + "aFomUST2wXw0hsi/OSZLEhRAi16AeUI0Y4SenblpEYtDAyhzFi77mqsceuF7QIq672m5yWq2qoGhPh7S+5lMEvuD8YLITzrC81Xc" + "b4HN3rxOvi6skmP4WJqZO8zPIw65WQUhRdhadhxM81HE+3InBToo1IhAF+11qt6hK0EuUf1guaGrUEDZRPAgomcOLBSdK6gQ9pn5" + "z3SiRPpw9SV4LWMqBjulGRD6K8u7XHXLPAiGZTF1cr/N2sEQuyri57XytugNISVcxh0q/Lvz0v0DyUAZM8H2ymbusxnKdIHanI9Z" + "US/XIXdia3PwqiOCgXDQaceOmotUnl8i0IXvSKwMjv3wB7UaC4YKyDurZ+gJJOzbjVjZ/mEf9h4JW1ZetNWVtQl2s+oZXCHOtVJD" + "2qLDyuMPO7EuafP8w7mbpKBTmw4ZAyCfrTL73FnEYhp6M3IEG+c4fdUCHMvT6ecG42DLKhdmWfCqzTdXmXFLj/bePo7YzfYpLOLl" + "hEzVdO5ojXPd6squqM+YHQS640UkdQ4v20UQ9ghFRkW1xUl7uPQkIAI0KWiDmaoi0YGFBxWpTs1U+wN2K6hfqgtJy80rl5ceUT7f" + "6B3MnwiIoAjzVc3n3TH6GodPIO08vrhrf4oz+G3OkdCaMHXURfzYkhji50/ThfAxgD7XDuzUYlUWGotbx0FiYEDBbi2b7t7L/w3f" + "P3nXYoSgaaHumXEnOAtugy6immasBFawtZq+GrOIYHJbH2tZhMpUysSTSnGF3BmqczIctDCwC82AbXU5sKEKFShEGnJSw9B0OlWu" + "Pa6kC/Yfapk6NzDyNB2ttsLhy0hCkT0BmgDn2P4h5T4PCVgLWUbA7LGQoYO5zbsUFBwtPUsVFK0yLyf/Gg9h4ebosT6NVh7AkzS2" + "riaNfm//lGUKK/8YDY49PAu0jpHp6JoXYg/z1+BjPvITdIQgf/8a0X36uqO+Yp2HZPzZzl/Ed8sC3ZadKxulNV9kkXN2Gc3v/VyB" + "ruICqdhHSXJkusQJw+vtClqbP2uT18wQN4SRLjFTC3/ZruElXJFipevCQcC8KNVkXW39qZNWJGmKAR15mprXot6VKXkRHfyQJJz0" + "lAMTaACA5LJoA1dDn7+UaikqUsfrlLE6ObbCasA5/wIp5zFsGvslVq/vJqGWk2mdXbas2fpWTSUECEB3gGoIFo9rmHTURX4cAALa" + "oZnjSkaKy1IdC3XDQsqT2vmAX3XjvaLhkCwWg56pst5LvS0AlcRGo49QHATNsY4h0SFkUmMyAGgS7+Pa2VwmCgWUoFFHRbO8ryoy" + "W57qONicyCxA2/OL28OrKGC/e25UsXwKBAXx+9uamcdxFvOQr9WqEJK1g4Q=" + ), + ], + }, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + }, + ], + "usage": { + "completion_tokens": 400, + "prompt_tokens": 246, + "total_tokens": 646, + "completion_tokens_details": {"reasoning_tokens": 339, "text_tokens": 61}, + "prompt_tokens_details": {"text_tokens": 246}, + }, + "vertex_ai_grounding_metadata": [], + "vertex_ai_url_context_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_citation_metadata": [], + }, ) diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py index 834094fdc78..cc542499a07 100644 --- a/tests/integration/translation/messages/bases/anthropic.py +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -6,75 +6,6 @@ from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGE from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE from pydantic import JsonValue -CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-opus-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-opus-5-5", - "max_tokens": 64, - "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-opus-5-5", - "id": "msg_011CfgC5HKvyve78CTFAw97f", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "anthropic/claude-opus-5-5", - "id": "msg_011CfgC5HKvyve78CTFAw97f", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, -) - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -209,76 +140,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-haiku-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-haiku-5-5", - "max_tokens": 64, - "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_011CfoboJT2cSWAvpvJEPTBy", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "anthropic/claude-haiku-5-5", - "id": "msg_011CfoboJT2cSWAvpvJEPTBy", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, -) - - CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -525,16 +386,253 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( ) +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSuQcKEAgSGAI4AUIIdGhpbmtpbmcSDKFGmJVRB+s8d6jjphoMrzic91ngVwSk+KnFIjALyD2lz/FJl0zwfGQJCf/81TXhJ1N49s46WltO" + "EwVctdFi/mNtBuz/H8qLT6nZnJIq1gbvKJzD4syhoHZof3EHkaAfHxNRootT8J6SwvRw82stIfRyxoH3+xjCQq6iZzkkQ9HaQvdEeCexjN8N" + "RFpwhhS/O183u+EESQtikc5PGw7Yzdwrl7YZ1wkQP6XVjyRrhHkqZHryYo1nf6G6g6jSE9eFyIc9abw6cBNwizPDuuXpcockqrQtpKabaO5B" + "o2GBtKltSR8M/aOCDwCKhCJfentm1d9bcRVgT5Nfksz33HjoD5ADev3d+NnM6co0fWZesi4brJPdN21JZW5LjaHkgIFc2NbnSsR9eQkzIx9g" + "G17kFtiCuaEzv25/G2gkTEbrj/aORXWQgQIcQQ0M+fBKDSFDfUy9tK3H45tEPoW6ej1c/oeYh+fCjJRgGone08Hp/B2F+f5YdAC94Y04+7bD" + "UyAm3RlHTPiHRq5HnpHChIrhGldYQ3QU10Etr0dWj2N9UL5BbnMGVDxrGV8K8zjxAlGTZbGQNnhNWHWt5pcMR8afGOv/1R25jK5sO74gbKAZ" + "HfNuWCBfGytBeyjM+yvYpLEaNROmrEX1uQvFPHmWxS/3x44BvJKHWvSrJ0LsDbP+eoOjFIzKma6HHqicJG8TraHIEpIaNF1XfAKyUWtXytlU" + "Uiv1hdVjWqWuEUalOXeF2eOWQENZGfX6A3X2v61JKtsAh8BEB0IIL6ztu9YnW1aVClJN3mBPYedtNLSMQfjc5CXhLQYsFo8q/OwCqCvmhc0t" + "SvZKH/j5beXTSGhEafdxWFLGilrEPE3qhNyHWGjW42/7n342t3kwE4TDt9MhnGuBQGOFtoO/YD/rbGJBq1nDYOBPhg/g6wJx4eMbY1+YpzmM" + "1TcJ07X/Zwd2NgbZ8YugR8Fkxbz+wNwLgm3Wx4TB5Parg78jdek0p5DlfZ1aP8jluIxnlvzBc2al3yrNlWX79OYQSJvcJsn2P74HCWjwRmsX" + "4lqhklY7/GiQP/dDFw8tPxFvfcU6AqAfv9OUICp0PvP3ATGhpTon06mQIo6rR6r7HUtGQdPUA1rAw2XgvUpoXl3f0l/SC2aIShpWjQUwu+Zn" + "1VLaGBnNfj5moS5HtzbKVW9IqU/d9HDRsTlI07JB4LuDleS1JCS4mJv3vQVX1rq1mk5ETNXW/JjrhKHlB8Vxm6cOghgB" +) + +CLAUDE_HAIKU_5_5_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg36zuYtPFhEEpad7wiAq", + "content": [ + { + "type": "thinking", + "thinking": "I need to read shop/validation.py to look at the edge case handling around zero, " + "negative, and non-integer values. I'll read a generous range starting around line " + "1-80 since I don't know the exact file length.\n" + "\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` first, starting at the `validate_quantity` definition " + "on line 18. I'm also checking the lines above it for helpers or constants that affect " + "how zero, negative, or non-integer values are treated.", + }, + { + "type": "tool_use", + "id": "toolu_01J9fZnNpvkQDk28Zj62Cc47", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 289, + "output_tokens_details": {"thinking_tokens": 117}, + "service_tier": "standard", + "inference_geo": "global", + }, +} + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "anthropic/claude-haiku-5-5"}, + expected_provider_endpoint="/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-anthropic-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-haiku-5-5", + "messages": MESSAGES_REQUEST["messages"], + "system": MESSAGES_REQUEST["system"], + "tools": MESSAGES_REQUEST["tools"], + "stream": False, + }, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + **CLAUDE_HAIKU_5_5_RESPONSE_FIELDS, + "model": "claude-haiku-5-5", + }, + expected_litellm_response={ + **MESSAGES_RESPONSE, + **CLAUDE_HAIKU_5_5_RESPONSE_FIELDS, + "model": "anthropic/claude-haiku-5-5", + }, +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSwQYKEAgSGAI4AUIIdGhpbmtpbmcSDE4DraHPnzJaL5XCExoMhZYoXkhSWfwTcx8lIjBt8d8pYV0VNCDIdfuGLWpioTySeXeUjAmK11R+" + "18SifTQpg+l0fRdIaI1HPrvqpikq3gX//sW77xYgKmFhC/pig1WvEWEErbbxw6XtLwFIuq+x62J2Vka5GeZeT5CBp4TcrFY718D90jSPqYm+" + "b1a/Zg+2njnVpSLFTFss7vNmNa1YiUND2W4cIODegkc1tRQ/SRZRslMzZ9Xh39VAu2FsjJ0A9W+r6619hN9/ixT2yfxY1LT2Vf1YtTGhyDvb" + "Cn8+GXmYfzCUiQKN1KzX8qjMoUIiaNVZgJy4kYTlqTHXHA+hkYe/I/dkoNc5Nb/arHCSBqmCW2/Jh4T7gTlokfS1YNIZbOMe4S+vByzm3r48" + "sjf+Wf4utdKOkZCizLWjTdKA37qPQq3EN8dMeqywLODKR/4SoxojI8t0d+0Shfv7xYP9kS679M6bUpBiLbiRofIMBR+tiBfZJIMRt+/36c4M" + "JKYYtYiU8l2xMH206+onxNvADUUBk/1WLE0AJ1up/5ZZJO3bXAubC/y3+2icxVCQ4wOjTd1N2+pCSQAD80yOdqNIYHokX9dvbFcgBP5SlpOV" + "1Ast2tXCpcfSsdd4dP2e6Rr0APxrFEevkWiss5EVxk/STjehW1Rq4WFoxhhlQ8wibT2zAbUnaeR1aAQW4JPBrZsC8RNs+tu1HIs++mq4lQC9" + "ls/pEn0L8QSQMbxtCGDHOmdjSDkzFuDZt01KqYKK+pnckSfhd2VxOLWXCtFZbxwviIisafYbL7ex8kOmazXXPrFBAsKLo84Y0ym2f2fmGnvS" + "9p+ibGgizHLF3WPQJVDBdxIn9GEszB2vCWq4zdYZ3HUs+nHFCTFU3FT+MMGtxrnSFKhVcgwis627+XqmsXP0oJbsv3Txw9GaC9v6jD1OKil0" + "WQpc1a/hTb3SwroE7x7gmEVe6QHnJjrrqG1tCWlpa9Dt6GZ82BCuU4326iPlZHdCtWOwvlLZqgreZM8RMPbdTa/rcfrGFcO/bT0q9yThzX7I" + "ecnuu8D28QJlT3Vp779WakHBeNgvNq05GD6dvhgB" +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQS6QYKEQgSGAI4AUIJbmFycmF0aW9uEgw+f8JSDrP1OzQWjmgaDC+5fzOQbkLuJcHCICIwMCi871N6wTZlw9YxH6IC09M+IrnPgjtTz7lc" + "SPv+bzBSksfh4tcVEvWKJdPcZObhKoUGONxWS5haAeeIk4yEZWsYuSP9a7PYWTyt0IhnYArjFQPGxT6p13RSod4N5u3QKo/iVz4tWT5m+vxl" + "u+w3IUtcC7sOChRgMR+cN/kaj7O81NCdTXlxAp6w8VshNjTVSKKCxcDP8C6xBE5Fa14FPTwQeHwqyOmNeSzta1vmof1SB3u3NUYb5YAYQ5YB" + "zszgyXpl8BakycxtgX8Q3V0jrS4D08YFXFCbPAEa6REPbcbDqGqnFnWPxOXoIpGAOXs21hk90asOqJH6AToFK4p0UWIMG9RFkvCbQ7otPI86" + "tb8Xx5LtL6SG0BAuYqKhB0JewYrmuOLT4OyzlVYWhUVn5aEwpJmR3qc900LkVmUA2SkyUMwgVPIgzoEMrPYWtmwABoMHjRg088WZB5FRmo7h" + "uf9VE4P5tUPUfHYipgMsK3h9WAuvV9a1+4YtRSB74woVdoDAIawRCwFg/T+Njl84mBr6mND5eTn6UmqNdqT0w3VucwLq6KChClA7ZnWItamC" + "dZwm8LtA9ysd/xuE20fonf1Y5DhW4KI7A/P4W72TLUwyYBc+1BAMy5IFYKGZiE8yXRMLDAUsYuW5YAbQua3sDFR1DInlNexNuExlVN9lI7BC" + "IS1tQKultivKpg2j9SJGDP1kFwwMMQzBee69ii3ACozf+kcnDY7Xch5yeLkKnZn1Qz0IMBPOkbe8kWBB1sVGYVXplDNuwNmE6SRwNIqeVdeo" + "ps86SpZ2bPXSk3H6614J2vVpV8SHQgoVv+7BZcsVXIKIAOhO3q/JQA835W+M7sESaAs1XFH54MfD+6UUlIhep9iq+3ck25zuDkoBT5xk9N3v" + "NeBm5JlUM33Bk8cIknUq35W/+jllBk5gEe+RQ1x85aNgTGzxIJDGhxP2tZG4pV8o0fkYR5n7PD1n0FbA3mJf8nnFcKDj0RevTz/y2woydaOd" + "S+OGnbuOtIJsJ2dpUFgDom9QDXNs1Bfx8PV29ND2bPOIYG+TqMP09e0zFvLVDF3Cz2+3zlHbaXj0xy44tD41+EpWYJcYAQ==" +) + +CLAUDE_OPUS_5_5_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg371xxoDKT2pPQfVcP6e", + "content": [ + { + "type": "thinking", + "thinking": "Since the user explicitly asked for the validation file first, I'll read " + "shop/validation.py around line 18 rather than parallelizing with the test file.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll open `shop/validation.py` around line 18 to see how `validate_quantity` handles " + "zero, negative, and non-integer inputs before checking the tests.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_011bPgnPBMCrDsv9vkTKQsFL", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 45}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 248, + "output_tokens_details": {"thinking_tokens": 150}, + "service_tier": "standard", + "inference_geo": "global", + }, +} + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "anthropic/claude-opus-5-5"}, + expected_provider_endpoint="/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-anthropic-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-opus-5-5", + "messages": MESSAGES_REQUEST["messages"], + "system": MESSAGES_REQUEST["system"], + "tools": MESSAGES_REQUEST["tools"], + "stream": False, + }, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + **CLAUDE_OPUS_5_5_RESPONSE_FIELDS, + "model": "claude-opus-5-5", + }, + expected_litellm_response={ + **MESSAGES_RESPONSE, + **CLAUDE_OPUS_5_5_RESPONSE_FIELDS, + "model": "anthropic/claude-opus-5-5", + }, +) + +CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0: Final = ( + "CAQS1gYKEAgSGAI4AUIIdGhpbmtpbmcSDGejrLS2VrnijeVfRxoM9JoxxlmEbYEIEjtoIjDZE1H+Mf2sV36GpDiZ6qmC9yPoLvEpR9SvLktB" + "uczCkKWCCiSMfKz3aswvNY+eX0Qq8wUeup4+/o2u5QfFgk6jxQgz4ZFZAlapXhbfwG2kcLegVjWxqnGBexyWZXd3Uu1l74YVsvig5pj8kIEJ" + "xKPkRb1J4Ts+q5bOF6sJjVLZLgtclDs0Qdwj73dUcqlrTqVlwWZP5jFxwjdzD7h19kZP7NEWFN8+tYKNjnNLiNYBq3RdnioxWPfsAC97L1rY" + "VcD1tHiRc3Wml/y5poBpvOGU/gDi96jlGELiZdqkYOPOYz01oI7snrYZZ+t1Um8FemhFBFSq1h2SSl7xihYB34cDPd7929obEKBoAexbPY2K" + "T+NtodCZ4DINMsj1dkAkZCx/QU27lOtRiPBS7JfgOvOXHFlfXho91r9tS9p4N4a1WQeTX96w83iGLXQK2DkcUChe22cUMdyRVzJn2hnoVN3m" + "Rl/bMQZxRiF0lcmq3fen6kWG3kbSpXIKCzbEZhfkRN1uVfZC1ed0JE8qgTYIUh4ZbXIDS1ZxEDbC086pZupahW5b00K7Q2bPuekv8QwIJsW9" + "G+FIdUYmMCDJuRQloY4TFYeNmCwOUT5ZMNoB0N0snFobDj744j8ME3IInjX03+FqWtRWJ2tsb0E0GvQlk9AisxHviaHJ0ejIwsgcdCtOeB8n" + "GWRlVh+GUTWoFtr75B6WN3jaIVpc57BmOWG0xiEqfJWDGSPxfMNx2FVQDF45PWBodL8hTYru0KSHaSD54h1txGFkJGeX8GDlLxe6B5xEYYpx" + "OU3+/4+DNtg7bsGvBRjtZ2TdzrE1N89j3hGlw/ShNS7tNarAfxwCLBbgKh/1uF3IcYfDt891lvvALQG1CupaXyyJiWxCr1oizeFlVe/FLSmg" + "H3yMwKVO8ol20DIlqrVjjqRWHH5wURpOVfNiQN8GMNAwRa20sUsbVl5aTTPkyVwYox+7s/QuW26zqdDztKasEToFU21SLyqSEnkxqNTTO2TN" + "55HYaq5g/GWqAaVIbfdHgwgbf25NL0CEmrWADytfBY4NCLEPNHTVCES5CKErwiGKkBgB" +) + +CLAUDE_FABLE_5_1_THINKING_SIGNATURE_1: Final = ( + "CAQSxQcKEQgSGAI4AUIJbmFycmF0aW9uEgxy5XyCv93dBoadwgoaDJlmUXHZn84IYPqUnSIwEAQvf5RKgehIVEOImDwlE5JQqhqPR9hP6TRD" + "c8VHaIIO0ajqrIvzmhC+qU5q+80vKuEGspcWI4tDdYEV8Op3nTUrfE1iU46R7zlVrkZb9bjdv+VZvamiz6RMNv0UAiQNaBOgA3E8qEa21LeK" + "nWB00LCD9lNSAtJQmLTEn9Eps9BLutgDKMBqMIKE9tpjBSDjJ738+6O3fGhc9+PghdrNUdiP2tvCIEHRDdVkSmSXO44BpqK4tbVaG08rZAwp" + "dLETvIpH8dgneIormOZWkW0vvlQ4tC47G2KZ861kXD4KAHxhfPzMET6yuaD2phSl/+BWvhNBgvCRqaDYf6LqExDZSy5XrIzt+SGzR8ZWpsVq" + "mY0QTAw5ECHlsj37Cilkw49OrY5zg9sQ1m7KuopASkT3gEes+TjPdofHms2Dgpfcx5svqF2kpCJBAj9ZaRqVVUMzWlIbrx2+QIrW9FR0wi2O" + "XmxdSrw2fjlp3m1ixjy/0jjyaKCfK0N09VTih3Q/OigXuDghsF7oXjjqYqHXxzRMkULGGjmunM6CcdJGcjzWRbSef2grS/KmsZZ7Xh/l/lRc" + "JXCZPaSHCgz8YwLSPOjRvEjMardae94UEXb/sKeAKczImcq5a0iVFvMjmUp7khBo+ygDh5uGmcos7+amUO3FiiSh0rxWbN20pHMGTsUURVXE" + "p2dzBvZ33egIaGVVjT5iQphQbMj/aY4IP8Mq/BaFqz0tvbffE4fv4bXz8VZYY9yxu6KxYLZYghzAvWyBujcZyK8Ju79ShlKpdTD2NE8rHZh8" + "X4IHKkL57qbcUf1onAxkEYt8fRR7o0aEuM+tZJX/vSorD1QAvuufraq6y8hP3pOtztVyRb1TFzqxwJwfaUavS2sKzPRf9DSguxK8aeaV0jXB" + "H66hLCfXE5bPB4+ez1LWqCPOXNEGVBPFE8Z+8VRXCkDGOyG3p428KKc+5o2qc6Wj6muqJayRenHO54TtEVzGc5h6vIhjHWT8TBEd6QktdByp" + "ino3ysKvuAEIT8jpPB+ucIMOaMZgMkJenSxKn7A+A/rNSQoLXX9uqpJDUE9+ZjucFABKApxej0o4PAsS5QSONURp+OUSjxJbEt4328Cl7LNW" + "MY8/X/fsyyxI20j5FBn5RzU8bHk9OWauNQO+vAf9/yGnMfLyjGZNRQKqVoURbO5/YwZ13y9pscRTOGxClYbG9XlilFluGnMeSCOPpsyufhgB" +) + +CLAUDE_FABLE_5_1_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg372jZvQrwtebLxTc8Ph", + "content": [ + { + "type": "thinking", + "thinking": "The user specifically asked to read the validation file first, so even though both " + "files are independent, I'll honor that order and read validation.py now.\n" + "\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "Reading shop/validation.py around line 18 to see the full validate_quantity function " + "and check how it handles zero, negatives, and non-integers.\n" + "\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_01RgnHAg9GpXpfL13hyRodfh", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 269, + "output_tokens_details": {"thinking_tokens": 171}, + "service_tier": "standard", + "inference_geo": "global", + }, +} + CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-fable-5-1", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "anthropic/claude-fable-5-1"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", @@ -542,395 +640,21 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, "model": "claude-fable-5-1", - "max_tokens": 64, + "messages": MESSAGES_REQUEST["messages"], + "system": MESSAGES_REQUEST["system"], + "tools": MESSAGES_REQUEST["tools"], "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], }, mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + **CLAUDE_FABLE_5_1_RESPONSE_FIELDS, "model": "claude-fable-5-1", - "id": "msg_011Cg33tMTaEU15ciNQxuvgV", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, }, expected_litellm_response={ + **MESSAGES_RESPONSE, + **CLAUDE_FABLE_5_1_RESPONSE_FIELDS, "model": "anthropic/claude-fable-5-1", - "id": "msg_011Cg33tMTaEU15ciNQxuvgV", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, -) - -CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-sonnet-4-6", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-sonnet-4-6", - "max_tokens": 64, - "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-sonnet-4-6", - "id": "msg_011CffzUNHaEfzVxCh5hskBG", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 18, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 5, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "anthropic/claude-sonnet-4-6", - "id": "msg_011CffzUNHaEfzVxCh5hskBG", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 18, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 5, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, -) - -CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-haiku-4-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-haiku-4-5", - "max_tokens": 64, - "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-haiku-4-5-20251001", - "id": "msg_011CfgDz9Vj8wXvKzhoM4MQA", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 17, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 5, - "service_tier": "standard", - "inference_geo": "not_available", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "anthropic/claude-haiku-4-5", - "id": "msg_011CfgDz9Vj8wXvKzhoM4MQA", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 17, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 5, - "service_tier": "standard", - "inference_geo": "not_available", - }, - "diagnostics": None, - }, -) - -CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-sonnet-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-sonnet-5", - "max_tokens": 64, - "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-sonnet-5", - "id": "msg_011CfgDzLXmJmnHRTpeBoDmZ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 21, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "anthropic/claude-sonnet-5", - "id": "msg_011CfgDzLXmJmnHRTpeBoDmZ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 21, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, -) - -CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-opus-4-8", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-opus-4-8", - "max_tokens": 64, - "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-opus-4-8", - "id": "msg_011CfgDzSFYYPC3aYE9rH98J", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 21, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "anthropic/claude-opus-4-8", - "id": "msg_011CfgDzSFYYPC3aYE9rH98J", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 21, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, -) - -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-sonnet-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-sonnet-5-5", - "max_tokens": 64, - "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-sonnet-5-5", - "id": "msg_011CfgDzY6H6Rnp7FC7LsZvm", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "anthropic/claude-sonnet-5-5", - "id": "msg_011CfgDzY6H6Rnp7FC7LsZvm", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, }, ) diff --git a/tests/integration/translation/messages/bases/azure.py b/tests/integration/translation/messages/bases/azure.py index d7239e209b6..2cd98350622 100644 --- a/tests/integration/translation/messages/bases/azure.py +++ b/tests/integration/translation/messages/bases/azure.py @@ -1,6 +1,9 @@ from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_BRIDGE_RESPONSE, MESSAGES_REQUEST +from integration.translation.templates.azure import AZURE_RESPONSES_BRIDGE_REQUEST, AZURE_RESPONSES_RESPONSE +from pydantic import JsonValue GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -290,290 +293,526 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) +GPT_6_LUNA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 388, "check_offset": 0}, + }, +] + GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "azure/gpt-6-luna", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/deployments/gpt-6-luna/chat/completions?api-version=2025-04-01-preview", - expected_provider_headers={ - "content-type": "application/json", - "api-key": "synthetic-azure-key", - "authorization": "Bearer synthetic-azure-key", - }, + litellm_request={**MESSAGES_REQUEST, "model": "azure/gpt-6-luna"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], + **AZURE_RESPONSES_BRIDGE_REQUEST, "model": "gpt-6-luna", - "max_completion_tokens": 64, + "instructions": MESSAGES_REQUEST["system"], }, mock_provider_response={ - "choices": [ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0534b0c3898d6d1b006ad023ba700c8195b63f1819ab327ab3", + "created_at": 1792025530, + "completed_at": 1792025531, + "content_filters": GPT_6_LUNA_CONTENT_FILTERS, + "instructions": MESSAGES_REQUEST["system"], + "model": "gpt-6-luna", + "output": [ { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - "finish_reason": "stop", - "index": 0, - "logprobs": None, - "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, - } - ], - "created": 1791231868, - "id": "chatcmpl-EVjWWeBeTmIBg80iLiSeFNeYYY20z", - "model": "gpt-6-luna-2026-09-22", - "object": "chat.completion", - "prompt_filter_results": [ + "id": "rs_0534b0c3898d6d1b006ad023bb09b8819590793fe299bffdcf", + "type": "reasoning", + "content": [], + "encrypted_content": ( + "gAAAAABq0CO7N77UmN8oIxFPRrP2rvE9LuqnrGGHsFzckseHLpNB24KIlg0OKnfemaCgYarrVksxeJBK" + "goiQ79xkF1NR9RI8-82w4OGNB0ViGNYNA7nf5kOqOTrMh1_gHecIN0Evxn8I0meiqOds7h1biKfkiZlm" + "19b-9r_Y2K--KXONVKFEbe8kP54_ka9KCxQ3_5XnL4IRfRS7lsLfoMjORKnapkVSnMjueRVVOprw_897" + "JJypeT61mzzBYM_18T6k68OE70JndMsr4GNRq4c_Lf_rSRoHg7_iFrdHsBdvMfN79dfp_PeLu5MZVRgc" + "ag-qM9aKUAvzhTD2-EutMw1C1ZNabU2S5Qa4RXq2OiU_iZljJI6JhpnJ9jT9g_M3W_0mklmFU2HBhQ6O" + "csIFmPePVcSaQlmofKgvq2ZThSHyFpLsT15ftBL2ZMZ3OWeodDnkfRr2fY5NDM7v7U1TLcAx7xCnP88W" + "03kyb3SuJGSsbPnYTGbNZN7ye3SBJRkCMpRgUfguSYdC46mmMjPdvdkLnTWHZ23GOoDG5A2lffUJZ4WX" + "NgPrMdG2FNThtHT0otHeDC75ahpa6i5xtqIsCGiFRVH_iSxhItArxFXoc7e6oBU9YxsgkUkyXW_zz6h5" + "h5E6DoJz4QckOAdL8E3SwL1nu6ovpS3_QgeiHa5RewKs6WclFCxeiwiyXO6bfjhl_VtOA8KyWqkO-rt4" + "T0RneV_bZHav6KoVanyCX-ZsAYj1WIhn4xEOXh6RdBjZJJB-yDw5cq6zdXkmN1hxs_nmA7hUemHkPclb" + "2lGF41N28IwMpcXq1rx2jrw2XleLnqFVKwgCLysinS9erU2vRN1CmmDQrGl7C5lIOrfSVCpB2W_p3BZW" + "NDKwAfF4i8uXCxy1_5-EjJ08CJXXYVchceOreHYNYxAXhVTHhJArQqVWQgTwJl9e_HhyA0OFZHojB54B" + "jlkMr7QJMm9VE9LIkOJgRgIrtUM2ULIy1tZ-EzuzEy9HYx5NuUKe_12ExKuUVrQmaJ2HRHvj_pPcmSk6" + "lOVrN5VvVBMj_KHd3MfkbUWVeVfzIsgWc149TexK6FPtvHGPxf_VQBdyA4Jl77dPp671wxH7puzy184L" + "V1IMdxq5eY1JG4QdCOluIDkkqbw_qYEiKGNXImGnp9LsXCcYa6JivYmSAw_vgZ5YhO98L49aLEI6nT3H" + "WRWuzY3H39aCkFdA8Y0DttzwXYvhz8UuenFgoGvIk0W7WSOLBxi6pSefdczpRizWLMm10zlrYVFvdMzU" + "PEx7OyDUhSXag61gYcsm6L5mcXGb2cojThdHEXcFZg36bQYjQVJzu_ASlvN-Z9AFNajIIyw8qKx3Q8pR" + "Vg3HCpQcSyRiDhicvT_Gp8d0X7VhoAhG_-LPg4AOz7U985JqP0wB2dB4sYhRgMZpQ29x7wgvgjNtsyqb" + "CA==" + ), + "summary": [], + }, { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } + "id": "msg_0534b0c3898d6d1b006ad023bb52f881958af6ee45425a3b25", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I need `shop/validation.py` next to see how it handles zero, negative, and " + "non-integer quantities." + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0534b0c3898d6d1b006ad023bb83a881958d95537bd9e16f3e", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":120}', + "call_id": "call_G1yFCqI4OnAJbTANskb1ROdb", + "name": "read_file", + }, ], - "routing": {"serving_pipereplica": "d20260930053954-f91e8c7b-prefill-r6-dp0-prefill"}, - "service_tier": "default", - "system_fingerprint": None, "usage": { - "completion_tokens": 5, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 3, - "engine_ttft_ms": 63, - "engine_ttlt_ms": 79, - "pre_inference_ms": 81, - "service_tbt_ms": 4, - "service_ttft_ms": 280, - "service_ttlt_ms": 301, - "user_visible_ttft_ms": 199, - }, - "prompt_tokens": 19, - "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, - "total_tokens": 24, + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 114, + "output_tokens_details": {"reasoning_tokens": 50}, + "total_tokens": 423, }, }, expected_litellm_response={ - "id": "chatcmpl-EVjWWeBeTmIBg80iLiSeFNeYYY20z", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_0534b0c3898d6d1b006ad023ba700c8195b63f1819ab327ab3", "model": "azure/gpt-6-luna", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 5}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 114}, + "content": [ + { + "type": "text", + "text": ( + "I need `shop/validation.py` next to see how it handles zero, negative, and non-integer quantities." + ), + }, + { + "type": "tool_use", + "id": "call_G1yFCqI4OnAJbTANskb1ROdb", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 120}, + }, + ], }, ) +GPT_6_1_SOL_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 475, "check_offset": 0}, + }, +] + GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "azure/gpt-6.1-sol", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/deployments/gpt-6.1-sol/chat/completions?api-version=2025-04-01-preview", - expected_provider_headers={ - "content-type": "application/json", - "api-key": "synthetic-azure-key", - "authorization": "Bearer synthetic-azure-key", - }, + litellm_request={**MESSAGES_REQUEST, "model": "azure/gpt-6.1-sol"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], + **AZURE_RESPONSES_BRIDGE_REQUEST, "model": "gpt-6.1-sol", - "max_completion_tokens": 64, + "instructions": MESSAGES_REQUEST["system"], }, mock_provider_response={ - "choices": [ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0a4a896afe94ce65006ad023b047248193999d7c3a0c4fec39", + "created_at": 1792025520, + "completed_at": 1792025524, + "content_filters": GPT_6_1_SOL_CONTENT_FILTERS, + "instructions": MESSAGES_REQUEST["system"], + "output": [ { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - "finish_reason": "stop", - "index": 0, - "logprobs": None, - "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, - } - ], - "created": 1791231869, - "id": "chatcmpl-EVjWXoW5I8J7jZD6gsZ3ONxzOqZ27", - "model": "gpt-6.1-sol-2026-09-29", - "object": "chat.completion", - "prompt_filter_results": [ + "id": "rs_0a4a896afe94ce65006ad023b19f8481938d51de793f83feed", + "type": "reasoning", + "content": [], + "encrypted_content": ( + "gAAAAABq0CO0QoINZ4oOVkvDvinFAJAMUBmzqqpXLoppdlyP4fFQsNqPOh-L08mQt8yf4OUtOwzk3E6Q" + "7CLHieco0s_tfuYHRlTU0Blr62KF_C77RScOjKYa6p4E2fjdwO44Y2HIiGfIo_GnUpkfwW1EiVZmj6L8" + "PO0biPqihvK3KJ-GNKB_R4-lLyKXrrvVN-985glSdHg-_fRrmyVGBvVxPFyERlD-WPm8fPujqMmXnUcA" + "v3N-wUoIEccd7PfJGqBs52UzllpH27mNCI71C_qbc2nZ97jZgSabTvAZKF8Q9zSJzWA_Gs72iQqd0X55" + "F5vD0-xrMuVdXFTYLV-y-S_1rN8tY1bEbdJWr7Dl84NvNWCYqA5byZgsMxAVyjADWXcVaagrZhNtQ1ol" + "II2oePJL2WyrKrOs4d7ZuGFW5BcFCDS5zxk6MwckKbLkSi3HM34z-AhIOjr6SNc5qcR8yur4Phm9pSJQ" + "qFscqQwtmEgbAzXqC9U5rwQ_c11N7o6Xc2bnoRXxeXQvKyGCPZWOqQr8AffHBCCOmeKjbPMV0ITek7Hy" + "RiOWg_Pjq-_xlVCoUGM7gIq5nq3HG1SqH7Yr6qev5Arq24UALiBbr0NAtmq3p-YGRgia3oDnSrnygIX8" + "AbqhnGrHz1kjBbwGr6fXF2XruSKuL96bQghfWGx6nhsbIDH4FI3akERb6Rh1B9XIWsWw4Zzo6-tg6Dnj" + "zfaJteZqIY4qS5mHSD9stXEVqu_CfnGh7EdOlZlnkG6kIQ3m6oAo2ZG9E58m32X-KEqZDjD4yUgWss9y" + "7cnFbYeEzntQneytoBpaTNXtSEsM1fUoz73Ol7FzqPI0CKIsp3UGje2K7LqAwkV54JrhqCfxkD5HmPJn" + "yEOnP0dODvyqD3CK-cbkoPoszI2Ba0dk11re5kF985he-Fjm7dXDNhQX93__nj4awDCtnX8_Ke5ld7O3" + "sA1ou7WHhgfGo-Jr4ZcGL5JWflPbI9oPpGOOvLsS-e8CUjTcTRb_eA2jApeGU2nAdW42U-CnXeScaR5e" + "JE1RoXTFXIP6MMSpxqpTRWYBfhIDyPEzwTveki-WOWgJtyNJ5W3GJxaEZla2uJ5_cdiD4-HgRy1UwxIJ" + "ODUh0F2hKC6niDVIZqoZ6Z4nb4OUvJP00_nGBA5-IL31Ij8I8U740RAZvnOsMOIsKelsJioM8Ii4MdPi" + "O2FYSdKjijEJmXLDdbYkQEXeqwfKGC7trXBOyRwO5AF6mLR9Yf8Bu_eiKbuf5DT9sw88P4JtfPi5LLIs" + "o2Ki44UnxePraO7vinqg7H7oOi-1gw7BbDaKH7tJs3vUOaxl6mA0BNTg5cDt6TtFZMpFgKNRyanWzSwG" + "WQGYlYbfUGVUnM_tjtgoBNFJAQz1s8fd-qOl1NQDggjlryl6TJk3RSiNYKyJ3fmpE10LM3LUtN3AoFMD" + "FbEEBstDuVg0zzvwe64zaKosn7PIh_Ym-y7QlMdcvUbsqO-T6pfsq8iqEz2anv3yujHkAXIaa3292lbA" + "dFIq80ycQSJbWNd2MtkYQHdWtWmQQoqMX1PdyJGp6N3GG45A4A==" + ), + "summary": [], + }, { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } + "id": "msg_0a4a896afe94ce65006ad023b2cc3081939533c6e664038345", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0a4a896afe94ce65006ad023b3229881938388d9bef2b4a8ef", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_5drWfeHZt2S3rfVa9SYefgB1", + "name": "read_file", + }, ], - "routing": {"serving_pipereplica": "d20260929014647-f3a90410-prefill-r11-dp0-prefill"}, - "service_tier": "default", - "system_fingerprint": None, "usage": { - "completion_tokens": 5, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 9, - "engine_ttft_ms": 477, - "engine_ttlt_ms": 530, - "pre_inference_ms": 44, - "service_tbt_ms": 8, - "service_ttft_ms": 795, - "service_ttlt_ms": 833, - "user_visible_ttft_ms": 750, - }, - "prompt_tokens": 19, - "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, - "total_tokens": 24, + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 137, + "output_tokens_details": {"reasoning_tokens": 72}, + "total_tokens": 446, }, }, expected_litellm_response={ - "id": "chatcmpl-EVjWXoW5I8J7jZD6gsZ3ONxzOqZ27", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_0a4a896afe94ce65006ad023b047248193999d7c3a0c4fec39", "model": "azure/gpt-6.1-sol", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 5}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 137}, + "content": [ + { + "type": "text", + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + }, + { + "type": "tool_use", + "id": "call_5drWfeHZt2S3rfVa9SYefgB1", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], }, ) +GPT_6_ASTRA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + } +] + GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "azure/gpt-6-astra", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/deployments/gpt-6-astra/chat/completions?api-version=2025-04-01-preview", - expected_provider_headers={ - "content-type": "application/json", - "api-key": "synthetic-azure-key", - "authorization": "Bearer synthetic-azure-key", - }, + litellm_request={**MESSAGES_REQUEST, "model": "azure/gpt-6-astra"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, expected_provider_request={ - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], + **AZURE_RESPONSES_BRIDGE_REQUEST, "model": "gpt-6-astra", - "max_completion_tokens": 64, + "instructions": MESSAGES_REQUEST["system"], }, mock_provider_response={ - "choices": [ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_019af305501e4822006ad023c3a50881958c4cf40623699cf8", + "created_at": 1792025539, + "completed_at": 1792025543, + "content_filters": GPT_6_ASTRA_CONTENT_FILTERS, + "instructions": MESSAGES_REQUEST["system"], + "model": "gpt-6-astra", + "output": [ { - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "protected_material_code": {"detected": False, "filtered": False}, - "protected_material_text": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - "finish_reason": "stop", - "index": 0, - "logprobs": None, - "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, - } - ], - "created": 1792022239, - "id": "chatcmpl-EZ38RBBvY1sBntKZmxDuJDJetT1Tq", - "model": "gpt-6-astra-2026-09-03", - "object": "chat.completion", - "prompt_filter_results": [ + "id": "rs_019af305501e4822006ad023c49ce4819590da413b9879e06b", + "type": "reasoning", + "content": [], + "encrypted_content": ( + "gAAAAABq0CPHdct5jJ3BolLC7QmXutcBsSZ0YzHiI8z2uK07jqwLifVhEROdF12ygdCVRtcrLrHPyHSh" + "pVbh41ULxn2L3SyVYLHbbkFMcwbzfky_kqZdUe_1D-PLxsVyI1KFMPLg0Fb9K-N9u_ElvkFcXXsmAhRQ" + "CcB4SE9TMe_5tKBHiZbqvOI112wig7cc57rDbvM0mEoKyFn3WgprP_ewHeCeP6DSEvVGDEMjUd_TIXAH" + "H7y2Ae3y0WRW7WUag_wAiO7xzAH13mh4S6eKeBTKh62krmYeUH3wETO8BF1WnkuZ_86lS0zWLBzeLJJn" + "JiXKXVquKatJ6HvO3m6FkPF0tBXhdc8YYtTF1B_WBMECARKcw4n3mPL3R6J2jpHe05Eo7W7uZkcY5E-q" + "mE_cqUK8YQXTXx-mR339iuAvqSe0N2Rgl9Ey1l7z0MBATIV0c6J4wRav6Di5aAxEEXkgDVk1jF9LZSHo" + "mcGjh2B2UWFiAZEELRDD7ovpP3c7B05h0QvwazXlZnskJf-xYZEHAl2CNZk0iqagFs6EtNrFguM5W2AH" + "6dsuaGjNGmdWM_kt4JqIUeScZLNgMty8qTOHFzd7tiBpuLkQgcnHzOigWb2vY_8ahy_zfzM2lVL-JQxP" + "9SB7Gq1m49Ln-byC1O_kPyqEFXELhhQ8peXQY5do2R2T-3I0WLOhDhP5rruLsL66vNHAAMjka6WmjI2g" + "oiy-JF_nVCplJd791Wqa3yf7Q7ksfasnSKtjR7-cgTtMcXX7n_5joMnqf4_LsqSq6gHdeKw4VTeT8wGB" + "Uo3MHLkgi8PnAIphdb3PKgifSjstE_oH4_ZbmlyCsHaFYq2aoy-S85ndlrR_PAHBtU8II-Ud_42EPzS-" + "Etdt5d1Qjjri3KTyPjPO7fAm8aPFr1k3GbjwASPMQ7cUs7yLhPmI-TCm5U-zgK1pIALviiU2o4IM7kdH" + "0-Dg1eyyZN8hsFGSHaUB7G_tGB0fj0H0GSztH8ltNjTK_276IOh8Jv5csxjjmlcMOHVBmDCkkE_DlVjv" + "NQtWQS9QBt9jd1gFSzaOhXVkKZJK3vTIaq8JxTvYtgLc8WM5pRHbnQr71TfWB-eLEYgN-aX5PueKkXGr" + "GauDpKkzVJyNZ6mZ_dVXwl1p5IiNNJ1xzhmZK2GVDFWt0nfOBYyxmCDfIxvQL8YAhXXAi1Hrjal82c68" + "lw7UBauyMojDOcbH7EDG9KNbVV-kmkhCW3aH0QCIALzHqO_cukXDqnYX-iVNuxWpW5KUAJdg9cfgwX35" + "O246HxDoSpoIYQ1GooqIWSImYrlNQajPHpJcpxnWeSgKOSYIAYCAr70MouYhUg-lsId7n2pr4gspPf1Z" + "UrBhAG3h4n9qSguKTlpnAYlln_2MzgVVN1cH5sopvgJJLr20HKS1cfu9lB-CHDF5h74z7q3ydZJljn8d" + "QLBqL3csr0ovOZkTjP21rOjPhdsedg4CLKnylgk5nTewbQz99lKxKPhfYK0-XxiDdE6aKvobQUoRupGN" + "Hv-gMLqJjehMUynaC6XzBwJ63pPNhH3AlCj3zSte0Mj1FkPQQIY2PjhMB49CmXBV8Z9YJv0=" + ), + "summary": [], + }, { - "prompt_index": 0, - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - "self_harm": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - }, - } + "id": "msg_019af305501e4822006ad023c5f73c819597a30ebb95c43472", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative " + "values, and non-integer quantities.\n" + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_019af305501e4822006ad023c65c408195b9fa8c35e75b1ebd", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_zAt0ZCuKW07iCDDK2RlX7lAX", + "name": "read_file", + }, ], - "routing": {"serving_pipereplica": "d20260921004409-52a9ff06-prefill-r28-dp0-prefill"}, - "service_tier": "default", - "system_fingerprint": None, "usage": { - "completion_tokens": 5, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "latency_checkpoint": { - "engine_tbt_ms": 12, - "engine_ttft_ms": 546, - "engine_ttlt_ms": 619, - "pre_inference_ms": 44, - "service_tbt_ms": 14, - "service_ttft_ms": 1023, - "service_ttlt_ms": 1092, - "user_visible_ttft_ms": 979, - }, - "prompt_tokens": 19, - "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, - "total_tokens": 24, + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 138, + "output_tokens_details": {"reasoning_tokens": 72}, + "total_tokens": 447, }, }, expected_litellm_response={ - "id": "chatcmpl-EZ38RBBvY1sBntKZmxDuJDJetT1Tq", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_019af305501e4822006ad023c3a50881958c4cf40623699cf8", "model": "azure/gpt-6-astra", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 5}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 138}, + "content": [ + { + "type": "text", + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative " + "values, and non-integer quantities.\n" + ), + }, + { + "type": "tool_use", + "id": "call_zAt0ZCuKW07iCDDK2RlX7lAX", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], + }, +) + +GPT_5_6_TERRA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 313, "check_offset": 0}, + }, +] + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "azure/gpt-5.6-terra"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={ + **AZURE_RESPONSES_BRIDGE_REQUEST, + "model": "gpt-5.6-terra", + "instructions": MESSAGES_REQUEST["system"], + }, + mock_provider_response={ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0d178c5d2401a624006ad023cf1c588194ab41d5803ed29401", + "created_at": 1792025551, + "completed_at": 1792025552, + "content_filters": GPT_5_6_TERRA_CONTENT_FILTERS, + "instructions": MESSAGES_REQUEST["system"], + "model": "gpt-5.6-terra", + "output": [ + { + "id": "rs_0d178c5d2401a624006ad023cfe304819486bdf040ec55eab7", + "type": "reasoning", + "content": [], + "encrypted_content": ( + "gAAAAABq0CPQDJzIdOV-2rO0aGYCUhKcJifPRv4ZkBaIrKZEARo0rk8CqAe8IV0c2kyBwOFe4M70GMM-" + "HrR1L3hIUxan9dcDB5_S7e45yDaxFMiKpbr20izrXNktdwBQfvYOKoMMd07fslVRR-GHOEtz8JvU98v5" + "81sxXgVTxNAWGSl_Kb22g2Fds02t1-1g2hT304vsi1sBxbA6P2b-NFNQ8N3NEfX6FoWi7UF0nH69-Dg3" + "sCMb4dpZ0gGZaZSHwvzXXHESEuiMv1nJSX18tfrFIt29gmw3Mq3sUxZ3eALkPZ54yDUlfYXl6sm2MHXT" + "2xn3l5x6jlG4X8hEqgdfHI76otxmfsXvgtbXCADCes87f00bkWatqWgq27oluRYsY02xDMhtrT4h4C4s" + "PIpV7PA9tdhS6Lo3znsQiOEQvMqnSS3dH1NKnZsAiyjNwVZVZa0opIzI5fpNCGjBmMv_4hB3jBRgenG-" + "Ku75OcwPp6HXMej9McvpLIKtXb8FMKS5wqiZzYBLp-F25hzOHI0QVl0pGFL5ya9mWU4hkQYeu6HuH6Zj" + "Sujzyy-8ZmlBjnRZSQp8mN4t34u2ofk4RtyuZKJkgjJfN2m5MxBTT3AccMalmbiEa7lJv7GV9kLcqR0G" + "T2Rzd6UMX6qPG92VdfjIgCj1Rzh1IWrJvMloKkMj5qB3nN2dABqfCtjBleStNoTjO2VDensXf11ceG9-" + "KKWeCjfrG0ONgTVXPjTTUPHR0BxtoixnRxIYBZFb8sKdYAPI_QINjby2SVhhQH7UtookBf7N7OqmIJKb" + "zR4CLshKnvthH4fJQAZdsfoEPR9I943uBpZi4HnF8mQPF61Aiw9B3bh3USBc9_Oh2rJL044B6qYVD3QK" + "KGydzcplRxQWA31m-4Zn9eVxn-xOtx4Tvpdh6BhlFPKX9p2W0mSnb7kjpzX51Ec0J2AsTNUOonaaMFt8" + "2MJDELTGcAb3tsAkPzzNK0M_zjQ43CCH5Cu1yV2WhbUzxCu4UM0spAIuzpAZv6JGmW1wF4RtAgLfJjc2" + "LxoxcS9MJbLOUPAYdRlwBTj_IL9PIDeNlGVRgVQYL-9xnNKiWD8Iyu2vj8AoslM41wVTEXvOAYyEs25E" + "yo9ia2km8909d42DjFi2_ngA5ofZI89KQHtCzGwfCf_VTLbpMmrOB4NDFVTUYQlAMBvuSsXLXJJp0QPl" + "n2UBHhZ3erIcQ1D990SFRdx1uY47-E2FnCCPljyhJrWHnFxTlGbuEY4i5vPOweqYzj-uBbLVeDbcfjCL" + "jSLFgM4Fa5Zg" + ), + "summary": [], + }, + { + "id": "msg_0d178c5d2401a624006ad023d011388194a221b5ec1cb6e8bd", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I need `shop/validation.py` next to verify the current behavior for zero, " + "negative, and non-integer quantities before reviewing the corresponding tests." + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0d178c5d2401a624006ad023d04f708194bb3ccd55f7afb1d0", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_xFsX3SKb9MppLeAL2L0aGZ8J", + "name": "read_file", + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 92, + "output_tokens_details": {"reasoning_tokens": 22}, + "total_tokens": 401, + }, + }, + expected_litellm_response={ + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_0d178c5d2401a624006ad023cf1c588194ab41d5803ed29401", + "model": "azure/gpt-5.6-terra", + "usage": {"input_tokens": 309, "output_tokens": 92}, + "content": [ + { + "type": "text", + "text": ( + "I need `shop/validation.py` next to verify the current behavior for zero, " + "negative, and non-integer quantities before reviewing the corresponding tests." + ), + }, + { + "type": "tool_use", + "id": "call_xFsX3SKb9MppLeAL2L0aGZ8J", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], }, ) diff --git a/tests/integration/translation/messages/bases/azure_ai.py b/tests/integration/translation/messages/bases/azure_ai.py index a7b5f3f5381..454a76400db 100644 --- a/tests/integration/translation/messages/bases/azure_ai.py +++ b/tests/integration/translation/messages/bases/azure_ai.py @@ -1,6 +1,10 @@ +from collections.abc import Mapping from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGES_RESPONSE +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE +from pydantic import JsonValue CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -69,76 +73,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "azure_ai/claude-haiku-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/anthropic/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-azure-ai-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-haiku-5-5", - "max_tokens": 64, - "stream": False, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "messages": [{"role": "user", "content": "Say hello."}], - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_011CfoboMv8t1SZhyzZeXDH3", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "model": "azure_ai/claude-haiku-5-5", - "id": "msg_011CfoboMv8t1SZhyzZeXDH3", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -482,17 +416,55 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( }, ) +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSqgcKEAgSGAI4AUIIdGhpbmtpbmcSDJhRZ6mJjwWk7Qp33hoMdnwv42chh3ronr/mIjA/ti2oyTG92CZPcqNiCgjiqzwhodkEUejDgKoH" + "fbOEwwLS6iSCqSKrLMCu9PQAp5gqxwZJ+Z/ZX5JUifYdc1pNfnYmlWpJJoBDLv30nDIpOQq73DNraWtbsW0Wm1ZWdrZAapLtVP4pPFrDlen3" + "3gj1ZOfbLh/3tM1yfQb3reW/p8c9DMc54gQTlDqVEm0wSuqtmk0epMxueMh9Vu97JxR2SbeG/kU3M22oqt2HqsDXIxC4z8GQfuhJVa4CFvoV" + "kPLudM9VrYwLzPQ9iOEVvjjw5fYFLf5KOBFI95+z0LHqBopKpPLeQFT9J4jO4h6jsRMkmeCK0GIPKAdvIl6XlFKEC8z1sUi8aIsU216w/8yS" + "HDQdw6p5pe2jwG9XZqs98xRfELsxn29J7WaxzaWI3YQvMxaLJG0ze1InomeGzBZi46QCqlW7dK0XwnDSNbkcnN/1I4JIJlJhnSVZsbirzgK2" + "B8EZXaloWxRSRzC4Aw4tJ3/hKw2R966UgQzp8vkr/Ezms3v0cnRjayopPCUHNcuHkuCuww2X69J22HcULSUcvEaQSLlVHxTw2cS4hOjhvXc5" + "ySJTZPYlKNQhaioYvELPq5ULD/k2hj9JZp+Go/fvIt3F9ysuC5PRnvgN8V357N0L9a79S8lbw2eituEWmjRZzOQujGdVfz+d4NU0v0Btihuc" + "/am2liAB+hY4JatYsXyRRx/tV9HcV5ZTjh5I8bV5NHC/MsP0dFWM9XmsRcxQ+X/SyNHWvZUaSzYd+oKtv4RLDvanfNdG8kIcSzrSJe6DdqY9" + "7d6DWU2/mcbIIBZm94Wv40Snxg/CEnaVB3SdK/E4cTmlUfvaRZvaUMWUSk4wPrIA5De3htk7d2AP97fv2QayaS0PlvdnwbAO+kOSmEEvBKq/" + "yI/0LoSrPCi08CRJ8rO2KNXv5spyCJAfLTG8im6KaeENj0M7+e/BWYVOlZfgqNqaXUu5a2EUlO/0QycPe0NEIZ70SdG54C1zkE9MTjzYoYXM" + "EFXaawPQ61dheOHClEyEbLlfKSa9IUX19hhht9GD6wJIjAGQ3A+YqS1HIqZEvI7gnkQlk19BWPHs4UCF8JhnTsJQ+KZLTYFO7kfX5shhPYff" + "DTlrCRGUrArdsM9ooNXDAqgEA6O0xMuX8G2eg13+4tmcZGv407LHLWtreo8fgrBmS1yWMRgB" +) -CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( +CLAUDE_HAIKU_5_5_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg37NEs5KS227ncBai4Sg", + "content": [ + { + "type": "thinking", + "thinking": "I should read shop/validation.py to check how it handles zero, negative, and " + "non-integer quantities before moving to the next file. I'll grab a reasonable line " + "range since I don't know the exact file length.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "tool_use", + "id": "toolu_012CKWjp8JY4UGL1TC5Lj79N", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 220, + "output_tokens_details": {"thinking_tokens": 120}, + "service_tier": "standard", + "inference_geo": "global", + }, +} + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "azure_ai/claude-fable-5-1", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "azure_ai/claude-haiku-5-5"}, expected_provider_endpoint="/anthropic/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-azure-ai-key", @@ -500,54 +472,79 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={ - "model": "claude-fable-5-1", - "max_tokens": 64, + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-haiku-5-5", + "messages": MESSAGES_REQUEST["messages"], + "system": [{"type": "text", "text": MESSAGES_REQUEST["system"]}], + "tools": MESSAGES_REQUEST["tools"], "stream": False, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "messages": [{"role": "user", "content": "Say hello."}], }, mock_provider_response={ - "model": "claude-fable-5-1", - "id": "msg_011Cg33vny3NNaurNqZaL94u", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "not_available", - }, - "diagnostics": None, + **ANTHROPIC_MESSAGES_RESPONSE, + **CLAUDE_HAIKU_5_5_RESPONSE_FIELDS, + "model": "claude-haiku-5-5", }, expected_litellm_response={ - "model": "azure_ai/claude-fable-5-1", - "id": "msg_011Cg33vny3NNaurNqZaL94u", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "not_available", - }, - "diagnostics": None, + **MESSAGES_RESPONSE, + **CLAUDE_HAIKU_5_5_RESPONSE_FIELDS, + "model": "azure_ai/claude-haiku-5-5", + }, +) + +CLAUDE_FABLE_5_1_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg37Nrg7YQTVeZkczeqqj", + "content": [ + { + "type": "text", + "text": "I'll read `shop/validation.py` around `validate_quantity` to see how zero, negative, " + "and non-integer quantities are handled.", + }, + { + "type": "tool_use", + "id": "toolu_01TCWoDbheT9e18mtVDH3xnQ", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 143, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "not_available", + }, +} + +CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "azure_ai/claude-fable-5-1"}, + expected_provider_endpoint="/anthropic/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-azure-ai-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-fable-5-1", + "messages": MESSAGES_REQUEST["messages"], + "system": [{"type": "text", "text": MESSAGES_REQUEST["system"]}], + "tools": MESSAGES_REQUEST["tools"], + "stream": False, + }, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + **CLAUDE_FABLE_5_1_RESPONSE_FIELDS, + "model": "claude-fable-5-1", + }, + expected_litellm_response={ + **MESSAGES_RESPONSE, + **CLAUDE_FABLE_5_1_RESPONSE_FIELDS, + "model": "azure_ai/claude-fable-5-1", }, ) diff --git a/tests/integration/translation/messages/bases/bedrock_converse.py b/tests/integration/translation/messages/bases/bedrock_converse.py index 3f70910f1aa..6a115160fbc 100644 --- a/tests/integration/translation/messages/bases/bedrock_converse.py +++ b/tests/integration/translation/messages/bases/bedrock_converse.py @@ -1,7 +1,14 @@ +from collections.abc import Mapping from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGES_RESPONSE +from integration.translation.templates.bedrock_converse import ( + BEDROCK_CONVERSE_REQUEST, + BEDROCK_CONVERSE_RESPONSE, +) +from pydantic import JsonValue CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -48,52 +55,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/converse", - expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], - }, - mock_provider_response={ - "metrics": {"latencyMs": 954}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", - "usage": { - "cacheReadInputTokenCount": 0, - "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, - "serverToolUsage": {}, - "totalTokens": 29, - }, - }, - expected_litellm_response={ - "id": ANY, - "type": "message", - "role": "assistant", - "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", - "stop_sequence": None, - "usage": {"input_tokens": 23, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -229,138 +190,444 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( +BEDROCK_CONVERSE_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in MESSAGES_RESPONSE.items() if key not in ("container", "diagnostics") +} + +BEDROCK_CONVERSE_MESSAGES_SYSTEM: Final[JsonValue] = [ + { + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect.", + }, +] + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSowcKEggSEAEYAjgBQgh0aGlua2luZxIMk7YkOhrvQi9zueMRGgz/wtD/BSP6B5NwRzYiML9G20YP9l1wNfsLU7KSBDRmwZRu6x21pAZl" + "Wi/ABO1zIPZcSdHWKXKYtMMiben4LCq+BscUTDnI69W7DmSmw2Z/AtQHNqUYtIqAwZ1KP/kuDEmOlF/q/QvdhYOm73hRzbJUjEmVkwU+yxX7" + "sXM9DB18oM7KbPLbIAHNu5cM0D1uQ7jKNmLv2w2Y61M0iv4Ff+hU2r8zCXHEGsBA3EP6OsfkmhI3CmUWkAorSt1UYPmjSNbW0sQG9qA3fhVN" + "leKcTQkVpKRV5rY2wcB0Y1OmLdCZCdSj3MXpmjL+Qxh2fW8nFQ1o3XXVIZNTgoJ43J95PtZUJrRMjQGPULeri4oHwXhZowtZmMbnNctjqolX" + "TqXHAqBZLR7UqtOvaAihr8blpdiU5LrWn8qGuV/j4Me25hjC68jW45xHPW/J3IGg5pNOCGNVCA0ZOFkFkHVjGUGjl0jL0KnjC95I1ngiRhdQ" + "DiEyfQq/ozP2nwgk7SD4Wdq5T3QtvJpBB7tLYSMiJOPR8s4gyIE0gjG1kLdOHcEROhz5sezE40sCftRKbishKYw3ZE3badsbkBgVVy7sYevj" + "Hg2E+FycNLGp7VeX3SZgn+zdn2y6ZR427yS7Fhh84PKQMIiX8YQUsg8M4ISBtv0gvqfcrODZeTGKcCprEFC4NNcf9DfunhE9WrVDNaaHAKql" + "/G3uTl6akwmUyAGWomY7YEGGDymxYYGzhucyYuJiyOC9ncLrXJftPadZvz302hOv/ng15Uh/jQwuWsWCs0BjRWEhkmVje76JbmK4Zau9EEx8" + "iy7V9ltWvB20iTn1pfgY40+QFv1j75n7yXD0/oJHehfkRqm8feEs0ERnB6C+iF9XAqBoRSlUh728bbxNzK+ogApwtEbBpmnpFz5j7xLJLG8e" + "a53TCKMyemzqv9bA3xNnenrgO/QYJ1o2heQ6Tyv01cop8EpF5nSqWIorCEbf7yp3f50ABdfZZSnT8Up5hAYHDhTJ/NZBVE8oQaQGbcGRuhl2" + "5sN156XvCS3GcEP4mHu1L9hoT+bfasIx4aCdwQMB86E5lSAtCjphz3m4sBTT67/7RqVf4ymvPxjEsZroz/pujM7oJN2QEvZKkvKIWpHDXPt9" + "efVjUpeQcpoSTuazPtnWVWbLvA4FvVXpnDP+Ed7bfkrXw4enrKqw0QYEbJpmGAE=" +) + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-haiku-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_MESSAGES_SYSTEM, }, mock_provider_response={ - "metrics": {"latencyMs": 892}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 147}}}, + "metrics": {"latencyMs": 4031}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + "text": "I should check shop/validation.py to handle edge cases like zero, " + "negative, and non-integer values before deciding on the next file to read. " + "I'll read through the file, starting with the first chunk of lines.\n\n", + }, + }, + }, + { + "text": "I'll read `shop/validation.py` around the `validate_quantity` definition at line " + "18, starting from the top of the file so the surrounding helpers are included.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "name": "read_file", + "toolUseId": "tooluse_fjdXCsIbwCrVLDv3Pzvn3B", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 853, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 853, + "cacheWriteInputTokens": 853, + "inputTokens": 4, + "outputTokens": 297, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1154, }, }, expected_litellm_response={ + **BEDROCK_CONVERSE_MESSAGES_RESPONSE, "id": ANY, - "type": "message", - "role": "assistant", - "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", - "stop_sequence": None, - "usage": {"input_tokens": 23, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, + "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", + "usage": {"input_tokens": 4, "output_tokens": 297, "cache_creation_input_tokens": 853}, + "content": [ + { + "type": "thinking", + "thinking": "I should check shop/validation.py to handle edge cases like zero, negative, and " + "non-integer values before deciding on the next file to read. I'll read through the file, " + "starting with the first chunk of lines.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` around the `validate_quantity` definition at line 18, " + "starting from the top of the file so the surrounding helpers are included.", + }, + { + "type": "tool_use", + "id": "tooluse_fjdXCsIbwCrVLDv3Pzvn3B", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], }, ) +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQS7wcKEggSEAEYAjgBQgh0aGlua2luZxIMTkG89+rWhZvnNujjGgy9ueQHNGYTXVgc2TsiMOfc/JR8YWRFuzvOfDDqCuTj51nOgpQyrrmL" + "r5fylFkkoYwi2WWTjeBjxS70RiBErSqKB3OzMtrLHgImTbinps+XcdvrRao0nkPqkXGo1FFfKLf1Tb3/Qytx1scxregT8j4LBtTirSlLJlUX" + "9zmQOi1DVInxJFYVl6p5KObZBKG8PSiVhdOwMxfnV/euyCWZO3JIZTywuOlsHhpb7k5QW6Ygn8Hciikz+A4dLMdBLdbFVT1y61nbTvGNITkO" + "0MngJu3DDe3XKO5v/WjMbNJHm7wEa40ZKaU5q6H7gMD/hJIhHuSWQjtNsD9A/uFBw/6o42gaEmndewGx593NobUz9hf5iWfmwezpqGBs+wsr" + "IMyie0VxW8KNwSR9dpkJfNAToId9EcDHVSlBGdGiQXlaz0Efed6PF57WgV6glU0HVyLKZ1sIDb1G1hhwTe6LKw6mxT/gcakl5S4GB6MJKvEi" + "vTVAqSWjlvSQPktHDvM4nwPejOt+HFcipmE1Wx8Gopfvg1eE1rwslB+wjUO1EOC31483lw5J14KOz1swzjiZcOTVj85CEFtadz8fPK5SCOsP" + "VNcdvpV8rh7KCdkDTnD7vTtS18Mu9QoOw0hl66l4O3uEuGa0wiQ1/0H2Mmul/+P+zMxrkYNkVicyDF7UQH9J22lPOaJylisuyZuW6Tamn7PK" + "sbhnTHbIeZpm37BsL4ZBN33aAtB5FGVjXevjt9/mJAuljybiFyA3WuoCov1/9j50vR+UIBW2XeqFbTqrfymqWzFiNWWM+6qGt6UR1S3o0wlO" + "2SXXVNhsNkXVRuR/4cJtjjpReVcrYL+LZNtFgHyWAMhRd4NOCLGQ4wQSeHRFyjqJA7xguls6spfcnBZXlgUFsdYWzyDBqbg8EANBfwI0cTQG" + "X0CZy/beSSxLl9YwDT6cHhQaZU0sJ7yl1AzUuVaf4T5IFDheDn9GKKNCjF6VdY7ycLaH9JfuKvPIM1tzioKcLoNDppFFGtHD+2bwHdw3NAT0" + "RQjuU7XLoUm9p1roY1m7RltcTl1nl/2K5G7x1//Fl4vN9MoyjiL5OF/vCoyaZuswAxZIQeDWxXvDBQVVZvTy5WA9qJmG8OX0QQM1+XKe84IV" + "Z/9lqwrTW3ktVSVuNsFHBFypo9KXONnd99ekwaiFyVeCs0W17Gt3qIwk+LEF0yuFEantdcMoJBMXa3V7oxTgu9a42mrpNOtVwQ2RcU2rLsGl" + "hvj4nDpPAnkNixpyX+SzCJ81FlutT1p5Wpw2GuL2gyZRfNpi8gJOYxgB" +) + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQSigYKEwgSEAEYAjgBQgluYXJyYXRpb24SDAwiB8XjpzUcjkdimhoMwIGz3+l5MeF2P8mjIjAhc1Uec4tzG5fzDJrcwUiHfLl9hpYALDND" + "+ykJZqmCUizSbZOB45cey71j2vi6fpMqpAWB5p+izlwQyOdW2tEZygJWj2ag/CSfLMDFz4FxatYYUcivtmo/ujCCBekTBt3hDhph2fE64sj3" + "xsueYPHqSuAdn2n30TPAxBXIR5coXHz6nTZpjouC2A6zXy8grDdO9jSeS1vPprkyCsJw17+jigjWDIkmrjUv8DR1s64GCfuV5dWQ2PdYBI/1" + "8o0OhR77veuPdOcNGcLfP15XX7vbtxJMP9LWOR5MLTgPjLjSAEL++eiisr52H4rB4Lvhz7n2KMfMClL7hqdxArHMpHh00wPi2NR8CGjKNTIP" + "HvI/3e13aZDzW8JEDnD22CMmRx7NLgPD8nuncPqoZdVE7jYOECQldD3iHT7Li+m7CAbimbSksM91rzSP2S3CR/FYGtkjqJP5dDlPuRuWVgat" + "jVM/xUEJl058GNMbTukUAFq2SStLHCCZh6upTBg9DtSNMmNi67602YHuQ+p8g8WRYIJh1PmkU8Y3u1coBp2LKQNuiq7MXT880DxtZ2Fldfi4" + "xnY4vkYyB8pylwb1MrhUxh/7xHEesoSO18+SB/rHosHX7BQM1zf8KIDP4wqSBJER0EOPu6h+u1wiD6VYsRQXFWGcZ2S77GyNhaIQnvcc8Sp1" + "sKoXtjNSek706MHLreobNqFfMZBTWtxcmYuzzYIQyPdJMKcdFHSsXLgaoV+5O2ZQmQbhm5Iw/pvLWYobWRXjwHbqR0tka6bzW0nTrX2REUbA" + "gUV3jAOuZKU2P7DtCDReemU6W0TJpsNIi1/2j2oYwa/Fhgne1WWhadCRN5tHbsAAdrwYbksR6mwIydsu8/on2QBYLRYiLNMN5CtbLMY4zZHB" + "sKzVE8MUAfiGKwhxBE3usWbXb+RhCo93uKe9RrCnikcdD6z53qQb24EpaAfm+rVvKzCElgZtGAE=" +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_MESSAGES_SYSTEM, + }, + mock_provider_response={ + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 155}}}, + "metrics": {"latencyMs": 6213}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + "text": "The user asked to read the validation file first, but since reading the " + "test file is independent, I can batch both reads together rather than " + "doing them sequentially. I'll read validation.py around line 18 (lines " + "1-60) along with the relevant test file section.\n\n", + }, + }, + }, + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + "text": "Opening both `shop/validation.py` and `tests/test_validation.py` to check " + "how `validate_quantity` handles zero, negative, and non-integer inputs.\n\n", + }, + }, + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "name": "read_file", + "toolUseId": "tooluse_1r54N0OeCEZFcknGLsKsaa", + "type": "tool_use", + }, + }, + { + "toolUse": { + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + "name": "read_file", + "toolUseId": "tooluse_VOzhTwpZcyOfeZH2KdVtDj", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, + "usage": { + "cacheDetails": [{"inputTokens": 853, "ttl": "5m"}], + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 853, + "cacheWriteInputTokens": 853, + "inputTokens": 4, + "outputTokens": 430, + "serverToolUsage": {}, + "totalTokens": 1287, + }, + }, + expected_litellm_response={ + **BEDROCK_CONVERSE_MESSAGES_RESPONSE, + "id": ANY, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "usage": {"input_tokens": 4, "output_tokens": 430, "cache_creation_input_tokens": 853}, + "content": [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, but since reading the test file is " + "independent, I can batch both reads together rather than doing them sequentially. I'll " + "read validation.py around line 18 (lines 1-60) along with the relevant test file section.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "Opening both `shop/validation.py` and `tests/test_validation.py` to check how " + "`validate_quantity` handles zero, negative, and non-integer inputs.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "tooluse_1r54N0OeCEZFcknGLsKsaa", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + { + "type": "tool_use", + "id": "tooluse_VOzhTwpZcyOfeZH2KdVtDj", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + }, + ], + }, +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSxgQKEggSEAEYAjgBQgh0aGlua2luZxIMaxDDsYU61QS1+n8RGgz156K29uKC1HWPNQkiMFcfXYkUyc+WiXLtgYu9mq4bS8cW7hn9MRw8" + "Sjvb5kO5dd7cwmt0bJHftIBtaLIANSrhA3JiSUmAaZwpmq3AgfM0LliKLK/KbQface14HtzKtd/5MF5Nil1AiqDmDblS1U03sFU5L3PyuCc+" + "hrSaftK7Z1ZBXIB8tOzsko5jeGex0cTsPHFrthM1cH1pay0NMmwU+r1ufrR4EaW7K30oyy3oLol3JpgNF433ra6tRB5EVW2Da1aGlJi6anmp" + "pp4Zq6VBJdGOFmZqTOQULt6dq5j1tg8pzcuQqkrYRipJREtdyxZSx8t98bdNrAWro5o4UW9cGcIZuWK24E994inMN0H4DAZliIHqpwAqwzAY" + "pbvild9GOu8lLpIMwyB2mlLADSzwK+J5/gxFSh2WZ1t56d5AlkGd55IywS7EB+AlZelbCCuyEDwJV9d3woohXmGUvEEuaT7dD6K7gHCIPcvI" + "nVdMnHzb6wlXmtKOUxl8mltIwm4PBBi2uoM8tgPq87fhXxGJgVOSzOdwhaJlbfkHCrVSXAloRx6qwvAZ8JcWBla26lKQo+4vxXo8Y4KgadCR" + "TMLNfymY5BUsz2eJa0fPm4k5CqR5updBjj8PIcUPU2FF9ZUTWQ3BwEhIJHiSdkSrRKhEy8Ifw8Cdzsbq4WxVlYWJ2H9sdycc43+qO9bgf+xB" + "gSeNP/G7uwmR2WcZocbodrIG97UYAQ==" +) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-opus-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-opus-5-5"}, expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_MESSAGES_SYSTEM, }, mock_provider_response={ - "metrics": {"latencyMs": 10460}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 35}}}, + "metrics": {"latencyMs": 3730}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + "text": "The user wants the validation file checked first, so I'll look around line " + "18 there and hold off on tests for now.\n\n", + }, + }, + }, + { + "text": "I'll open `shop/validation.py` around `validate_quantity` (line 18) to see how it " + "treats zero, negative and non-integer values. After that I'll check the test file.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + "name": "read_file", + "toolUseId": "tooluse_g2htBkaHk6egShGW6MsbjH", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 853, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 853, + "cacheWriteInputTokens": 853, + "inputTokens": 4, + "outputTokens": 190, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1047, }, }, expected_litellm_response={ + **BEDROCK_CONVERSE_MESSAGES_RESPONSE, "id": ANY, - "type": "message", - "role": "assistant", "model": "bedrock/converse/us.anthropic.claude-opus-5-5", - "stop_sequence": None, - "usage": {"input_tokens": 23, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 4, "output_tokens": 190, "cache_creation_input_tokens": 853}, + "content": [ + { + "type": "thinking", + "thinking": "The user wants the validation file checked first, so I'll look around line 18 there and " + "hold off on tests for now.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll open `shop/validation.py` around `validate_quantity` (line 18) to see how it treats " + "zero, negative and non-integer values. After that I'll check the test file.", + }, + { + "type": "tool_use", + "id": "tooluse_g2htBkaHk6egShGW6MsbjH", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + }, + ], }, ) - CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-fable-5-1", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-fable-5-1"}, expected_provider_endpoint="/model/us.anthropic.claude-fable-5-1/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_MESSAGES_SYSTEM, + "toolConfig": { + "tools": [ + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + "name": "search_files", + "description": "Search source and test files for a literal term.", + }, + }, + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + "name": "read_file", + "description": "Read a range of lines from a project file.", + }, + }, + ], + }, }, mock_provider_response={ - "metrics": {"latencyMs": 2352}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}}, + "metrics": {"latencyMs": 4888}, + "output": { + "message": { + "content": [ + { + "text": "I'll read the validation module to see how zero and negative quantities are " + "currently treated.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "name": "read_file", + "toolUseId": "tooluse_ykxX1UplXkgne3EfgDI4Cv", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 873, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 873, + "cacheWriteInputTokens": 873, + "inputTokens": 4, + "outputTokens": 125, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1002, }, }, expected_litellm_response={ + **BEDROCK_CONVERSE_MESSAGES_RESPONSE, "id": ANY, - "type": "message", - "role": "assistant", "model": "bedrock/converse/us.anthropic.claude-fable-5-1", - "stop_sequence": None, - "usage": {"input_tokens": 23, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 4, "output_tokens": 125, "cache_creation_input_tokens": 873}, + "content": [ + { + "type": "text", + "text": "I'll read the validation module to see how zero and negative quantities are currently " + "treated.", + }, + { + "type": "tool_use", + "id": "tooluse_ykxX1UplXkgne3EfgDI4Cv", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + ], }, ) diff --git a/tests/integration/translation/messages/bases/bedrock_invoke.py b/tests/integration/translation/messages/bases/bedrock_invoke.py index db3b5dfd292..f5fa4192284 100644 --- a/tests/integration/translation/messages/bases/bedrock_invoke.py +++ b/tests/integration/translation/messages/bases/bedrock_invoke.py @@ -1,6 +1,13 @@ +from collections.abc import Mapping from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGES_RESPONSE +from integration.translation.templates.bedrock_invoke import ( + BEDROCK_INVOKE_REQUEST, + BEDROCK_INVOKE_RESPONSE, +) +from pydantic import JsonValue CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -60,67 +67,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/invoke", - expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": "Say hello."}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_bdrk_4eoumg4voyx6qt55ft5ts2sze3seufnqcvaqbdsjwfyd67wehaoa", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - }, - }, - expected_litellm_response={ - "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", - "id": "msg_bdrk_4eoumg4voyx6qt55ft5ts2sze3seufnqcvaqbdsjwfyd67wehaoa", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - }, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -299,181 +245,363 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( +BEDROCK_INVOKE_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in MESSAGES_RESPONSE.items() if key not in ("diagnostics",) +} + +BEDROCK_INVOKE_MESSAGES_SYSTEM: Final[JsonValue] = [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect.", + }, +] + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQS0AUKEggSEAEYAjgBQgh0aGlua2luZxIM9tsaCdqj5UGVQgHOGgylOg8RDlmh3f1L79YiMFFshwFAzn8fQTRWSqaR7dpBkcadG+cFDccq" + "6Cu6MZxJNVXFtqpmgsWFQR5WGlEgKirrBMjA2IESdj0ySte+0ETFtNXPDkfiovCk9en3WHVYW+jvrAIJzOA7Y9oF52rkQUYY/oIeksjLdriJ" + "3dTogQyx1PR+LkdBZp63sAr+Zn92Q/7/ZBj4k8f6/wlICa/KiRtklkGz/KfLpCJlhzCoxDZ9nqaMKeWLHBX5/UlJ7Lf8j4VrW4Tr9oQ3VRia" + "iue189vHibmvLtb0lSOoiMq4hEvE2dA0x7RZnLXQ+mULWd7TVnEeQ1NnGt8SNV56ru58gCU6t6Z/KylY3K5H0EEkFImpOigKlNrdDrk7wRxj" + "hCaFKsc88HUJ15V+3iA0ys3IcZarEFeuDj3fjiKYGtDzaWWDGpmQs7CsKVjAs6YfVh3Eks0KGuDppfnM8QyvmUz8y7aIkeuuES1u9F82g3NQ" + "3FoReR4wwdy76xypUHY415wLODfAiSoJjqsI9DArNQc//8UeQX++ECLuvwNm72CgbcN7L13BznbeSQdV1Un65uXS2wT+1eGvDBgGuwuXgvY5" + "ri2U5YgvnowPboiziBMaUxKNEhNouvD+l+p1bxalC03JcCUCA1MJzqAafufTscOdR1mTSl5V2+PPHz964pvxDCMRR0n+nBGsc3obeoS+9Vqp" + "yRG2lrPQj8t9GYzL93C2gbB3o0mdBB2dZTlKPxTnkCanL+1PwLjuKiy7lklHNo/upTP84kISsX+pOSrSNRW//BDbC5wkqgXHx/GsVL9OfrBe" + "P7f4SmUh6VWykR44kEO7vfR2LF1N3LRsHrI4OuNXlh6AbxShwGMvdimckzK5IjCTPX6DKQBXWmlmNJyeNzw2U37e44S9+vhdz5TBhfcYAQ==" +) + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/invoke", + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": "Say hello."}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", + **BEDROCK_INVOKE_REQUEST, + "messages": MESSAGES_REQUEST["messages"], + "system": BEDROCK_INVOKE_MESSAGES_SYSTEM, + "tools": MESSAGES_REQUEST["tools"], }, mock_provider_response={ - "model": "claude-sonnet-5-5", - "id": "msg_bdrk_g36x73tqvgb222xutjv3lkmjrff6fbfmtc23hkagpgosftu4tjda", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + **BEDROCK_INVOKE_RESPONSE, + "id": "msg_bdrk_wzz4hwc4dbdg5xyblodt4ucbo6racirmvwxrmm7knfhbqy4m2tfq", + "content": [ + { + "type": "thinking", + "thinking": "I'll take a look at shop/validation.py to check how it handles edge cases like zero, " + "negative, or non-integer quantities before deciding what to review next.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01SAJFKxZ5AbvBAPai5cpFSs", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 185, + "output_tokens_details": {"thinking_tokens": 85}, "service_tier": "standard", }, }, expected_litellm_response={ - "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", - "id": "msg_bdrk_g36x73tqvgb222xutjv3lkmjrff6fbfmtc23hkagpgosftu4tjda", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + **BEDROCK_INVOKE_MESSAGES_RESPONSE, + "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", + "id": "msg_bdrk_wzz4hwc4dbdg5xyblodt4ucbo6racirmvwxrmm7knfhbqy4m2tfq", + "content": [ + { + "type": "thinking", + "thinking": "I'll take a look at shop/validation.py to check how it handles edge cases like zero, " + "negative, or non-integer quantities before deciding what to review next.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01SAJFKxZ5AbvBAPai5cpFSs", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 185, + "output_tokens_details": {"thinking_tokens": 85}, "service_tier": "standard", }, }, ) +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSnwYKEggSEAEYAjgBQgh0aGlua2luZxIM5Rdi0bxwZPzY53K/Ggz5v0FGebtekQAdYi0iML9pU9O153URlvQmE61tNy33MyzsPW3gebf0" + "uYHHYrfZvthRnZNKNSSJpftkUOnBHiq6BQpuaRBOmr5jlQmx/nra8YP41PjKMwpGTJHkjh2COy2g0Ex2ZJk2Zi5A4jdms8plNf0vG3bXg01s" + "woW+wMC8KnoLBoDYQ5rrR+kSVwQUUir1/UMP0Si3Ep96qLJ9923Tf8ypQe/lTP5Ta/iWJ8jEc5SgMp/7HquIvNxyz3efFSyhUF/82y1WwPdQ" + "YcITad5XfHA/gz/fdCOGCmYJndI6aauKBHXsq4dk+B3ZDbSr1URb29NolM2lWfHb3FngvRknZoHuMX7bVz6hrt2gsSRUoEaME/JNvf+L430t" + "h/lhfk5dHljt8YVf6wI4ZMAwty5ViM43M4In1AEaTJ4VEd9VfYgZ48zpxRqjRFXQvBORAqoOEempYRDphZZcPuxjbrcqW2fzwMDH+rfMZlGu" + "ssDEySgfxj3/ODN0W9US6zdQqJ1YgaOBTH+TEEmcsw0Y5Mq1qioYPeLJjVj9FVLjpwf+9xGBUGoTi5+20T6/o0R7KOMl7iUx7I7HanKoIHHQ" + "uAl/eFlBNSB7nlv7L7i1VC8VFTQ83mk9vVyg74guzcidr8QNn6lCFh8b2sGEUj5uEYLoRFhSAa7uw0ja6cyLOw/62OwXKlJaoV8Yy2awvYYY" + "/ojYlskRB31Ux+wVcSX4J4RqKUwz/067iQecmpXYKMSB6mxc6ig9oyZYbqNd0w4x7L129svgFAY0k3cDS2jUePia9Y+SXJgCNXgjU8BLMj82" + "NUy6WNtjCBXb89yxHWEhXdyNQEtOruc/zWDM6pdHnMsbbgK8q+anyttH6wFsorXwLfJnPTIR4dYujPxRph+ac52uHbhMQKtNB9N0Hd8kT8E+" + "kq9RXXwYjGoMj3CGc5KN0PELEpujCkx6omfN+6yMCJntks/78q4Zv1coVeb4SP22doPtMyjPR1wbBVMCoM+6cPF766uvFCg+J2oDGAE=" +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/invoke", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + **BEDROCK_INVOKE_REQUEST, + "messages": MESSAGES_REQUEST["messages"], + "system": BEDROCK_INVOKE_MESSAGES_SYSTEM, + "tools": MESSAGES_REQUEST["tools"], + }, + mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, + "model": "claude-sonnet-5-5", + "id": "msg_bdrk_rcnbkubesv2zarg2r6uit3hne2znqytllke3i5jo6sm6eh7ml7za", + "content": [ + { + "type": "thinking", + "thinking": "Since the user asked me to look at the validation file first, I'll start there and read " + "through roughly the first 60 lines to get the context I need before considering the tests.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I need `shop/validation.py` first. It defines `validate_quantity` at line 18, so it will " + "show how zero, negative and non-integer values are handled. I'll read the tests after " + "that.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_012x6zRfbMwPWH1TvwvtJcTd", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 257, + "output_tokens_details": {"thinking_tokens": 98}, + "service_tier": "standard", + }, + }, + expected_litellm_response={ + **BEDROCK_INVOKE_MESSAGES_RESPONSE, + "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", + "id": "msg_bdrk_rcnbkubesv2zarg2r6uit3hne2znqytllke3i5jo6sm6eh7ml7za", + "content": [ + { + "type": "thinking", + "thinking": "Since the user asked me to look at the validation file first, I'll start there and read " + "through roughly the first 60 lines to get the context I need before considering the tests.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I need `shop/validation.py` first. It defines `validate_quantity` at line 18, so it will " + "show how zero, negative and non-integer values are handled. I'll read the tests after " + "that.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_012x6zRfbMwPWH1TvwvtJcTd", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 257, + "output_tokens_details": {"thinking_tokens": 98}, + "service_tier": "standard", + }, + }, +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSigUKEggSEAEYAjgBQgh0aGlua2luZxIMo+bg7J1athsROSzTGgz8CIhzKVCiV3U3zR8iMFgRZ0GxT3RyB90ZBPfccjWj9SmeP5Su7x5K" + "v4XcUof6Yw7d+Q31UhW8CTZdeivdDCqlBOkneYlYuN0wm6pkFQG8Cg+ThPVN8HGn/QiZPBihOr5ZJK1joYur01TDHIYWzKuBOHJ/fmYfdgoG" + "I0d6WAsw3FdL4zsbhWSEHbbD8+RFz/wk/LqOdaQNYS4I42fytuO9+i5bVthVB3UVl0QEDxPw9s/4c/RaFMQpIyuCigqyJpZC6UKFCtAPHgPW" + "ASOJtP7s0Yrv+S6MBqJOP3oic95i1IPy873PVLJu+mmyKp8yWRqiK8EN4oNE1HuXKqpvPhm+aCKQK7rUJgtoTxNfpiGeZCDaA8am0ysM1M5y" + "2Zi1V8N4sKDz2+SbEDF8EaHp6eicIZ9f6Z5xII7tR/xS2gP7ZNvSt+HNC7pYoQ5rA9xEWO541RvtcloVj3D7OWeaCLgzP3rsp+XIrNMw5017" + "+98q2G3FAHiCa25VLM3nUyxcG9yi6t6AQFl4ISByj6oh22xrcQJ7b+fiLabG48PtdeVZip/LUAeWm1Dw6e3Vl9kmRZi07/gT+bCp3FhnOHSW" + "bkZKuWB0LWEWa1dBWWNllTDgc2t+jnOL9hVu3j92quUcauOQXA1de746z0l2A8M0bl+Vqf4dWrBk/rvdu5uumIUqgEk3YM8ClKy7m5ylOmct" + "QI8Z9Rsuvt+ianHftrp7VckHdX4BFxmvvtjC+7NljF+aOUJbZpasecjeLT5hf72Xh/OTxKCoylsxrPYbFvT06ZAM12sLHiQ4II+FtF9nXvQl" + "xBY7c4C7qxgB" +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQS7wUKEwgSEAEYAjgBQgluYXJyYXRpb24SDIDG/3Oi4Lc/e5FR9BoM/dTQTa3LvgbUByp9IjBRgDtuGyPNi3LkPeVXyLT98JrhBRqJktyj" + "WMovEVaNb0VsiF+Q3xFd1i/Mtp/aHX0qiQVkW5ID1q6VrZOWTLrM7vI2Rz4WYUV8fOSa7n8kamwNbwrEKY+uQ7vsgjge+f0Yr9bllDitTOsg" + "P19+CxdUmgLXTva7uu5w44S+KtSj+sSmYVW8Wl/S1DWBKVxL/8pXeJf6t6vQlHMu58EYfW2pY43v+7xUPNBRU+l4+UfBz4DRZHAJpQq82e5z" + "QHjdyU6nSqcxlE1zDrAmmiSiCskbg6CwH0VAredEgdVwRFrdCK+A9NiqzDRSASmZu2VYoPZwgWdd/ufIJ2oqH56p+6JszgX92U0d9yJ8o6uw" + "fvFrP08YYayBhJn7Ba2lpDTdlCtYLT9qsLyXwQ4NIwPr36nzBbDWhRhF5Z5sGdA8vRnyue/CYpbLdI8Qg3JiXaXXTsGhDvbAmX6x55PhXiag" + "WaQSsQ6UzsJIdSUdjeD5bnrJ841oSnsG5qKaUfz+yr3cRn4ZnIHqDbZ4o9o/0lQC4++WHe2VddIIET3vVITKF+jKUsHo62vr5nx6tyUj5dGj" + "ZMyFsEMYmf4b9hgij84/mgCOGLNQ5Nu1PM1GKnMpm2dEaosjmSB14pTkuJCzUWGrLANWcgKJRRfv1Wy6n1xaORcm8w7mIWaYbqcxBPddTVOi" + "sXQ2Gr5pZRsbmcMnYcsDF/LejdLT7m1w4WEeOG7B7kXue2FkVaBq0k5K/mgGuIpdmhuwEYuzA5nf+E91Ik0+ktTT5tX/LfhlTc1RqIT07G86" + "ufm8yKpp6Uc7PvWXGm7tR+/70sN5O4ltSiamehskjtAHj1ev3zc/dQJ3oQP6ljf1AY7/ZB9v6YgLY1c0tZ+cHUQ/0+ND+7sraZu/Xiihggfc" + "s8SMejElcBxTWV86zG2ijctI5UlotIclfNMiGAE=" +) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-opus-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-opus-5-5"}, expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": "Say hello."}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", + **BEDROCK_INVOKE_REQUEST, + "messages": MESSAGES_REQUEST["messages"], + "system": BEDROCK_INVOKE_MESSAGES_SYSTEM, + "tools": MESSAGES_REQUEST["tools"], }, mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, "model": "claude-opus-5-5", - "id": "msg_bdrk_lq3occizkeecj263kns2j73zgj4lnn3kzlgll5xnqvk3etlf4pqq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_twd7a5k6nmlhwudxctflvx44se2jpvhjrwimhwul2gwzgnni3zsa", + "content": [ + { + "type": "thinking", + "thinking": "I'll start by reading the validation file around line 18, as the user requested first, " + "before checking the tests.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "Now I'll check `validate_quantity` in `shop/validation.py` around line 18 to see how it " + "handles zero, negative, and non-integer inputs before reviewing the test.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01Kf2DAxymu3xTnb2AwxZZgB", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 221, + "output_tokens_details": {"thinking_tokens": 50}, "service_tier": "standard", }, }, expected_litellm_response={ + **BEDROCK_INVOKE_MESSAGES_RESPONSE, "model": "bedrock/invoke/us.anthropic.claude-opus-5-5", - "id": "msg_bdrk_lq3occizkeecj263kns2j73zgj4lnn3kzlgll5xnqvk3etlf4pqq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_twd7a5k6nmlhwudxctflvx44se2jpvhjrwimhwul2gwzgnni3zsa", + "content": [ + { + "type": "thinking", + "thinking": "I'll start by reading the validation file around line 18, as the user requested first, " + "before checking the tests.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "Now I'll check `validate_quantity` in `shop/validation.py` around line 18 to see how it " + "handles zero, negative, and non-integer inputs before reviewing the test.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01Kf2DAxymu3xTnb2AwxZZgB", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 221, + "output_tokens_details": {"thinking_tokens": 50}, "service_tier": "standard", }, }, ) - CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-fable-5-1", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-fable-5-1"}, expected_provider_endpoint="/model/us.anthropic.claude-fable-5-1/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": "Say hello."}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", + **BEDROCK_INVOKE_REQUEST, + "messages": MESSAGES_REQUEST["messages"], + "system": BEDROCK_INVOKE_MESSAGES_SYSTEM, + "tools": MESSAGES_REQUEST["tools"], }, mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_bdrk_dljkl2he4crmo25l5bqtvtdl4fesopljelt4povcvsjcln2vntaq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_gh2vug2tpleguniqdb6ry6otz5y4x2jxg3lneudi2hx4wrv6xp5q", + "content": [ + { + "type": "text", + "text": "I'll read the validation function around line 18 in shop/validation.py.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01Wr8tPMkYit6TSkCn7oBKcr", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 122, "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", }, }, expected_litellm_response={ + **BEDROCK_INVOKE_MESSAGES_RESPONSE, "model": "bedrock/invoke/us.anthropic.claude-fable-5-1", - "id": "msg_bdrk_dljkl2he4crmo25l5bqtvtdl4fesopljelt4povcvsjcln2vntaq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_gh2vug2tpleguniqdb6ry6otz5y4x2jxg3lneudi2hx4wrv6xp5q", + "content": [ + { + "type": "text", + "text": "I'll read the validation function around line 18 in shop/validation.py.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01Wr8tPMkYit6TSkCn7oBKcr", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 122, "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", }, diff --git a/tests/integration/translation/messages/bases/bedrock_mantle.py b/tests/integration/translation/messages/bases/bedrock_mantle.py index e932e03ad5e..7524bf8f7ea 100644 --- a/tests/integration/translation/messages/bases/bedrock_mantle.py +++ b/tests/integration/translation/messages/bases/bedrock_mantle.py @@ -1,7 +1,13 @@ +from collections.abc import Mapping from typing import Final from integration.translation.case import TranslationTestCase - +from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGES_RESPONSE +from integration.translation.templates.bedrock_mantle import ( + BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, +) +from pydantic import JsonValue CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -456,176 +462,351 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) - -GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock_mantle/openai.gpt-6-luna", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/v1/responses", - expected_provider_headers={ - "authorization": "Bearer synthetic-bedrock-mantle-key", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "openai.gpt-6-luna", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - }, - mock_provider_response={ - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791240608, - "created_at": 1791240608, - "error": None, - "frequency_penalty": 0.0, - "id": "resp_o2at36wcxwspffhlmvydms2hpl24bse46blsmsm52ov4xtt4ordq", - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, - "metadata": {}, - "model": "openai.gpt-6-luna", - "object": "response", - "output": [ - { - "content": [{"annotations": [], "logprobs": [], "text": "Hello!", "type": "output_text"}], - "id": "msg_333c149682be50c9b9baebeecbee8f18", - "phase": "final_answer", - "role": "assistant", - "status": "completed", - "type": "message", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "in_memory", - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "safety_identifier": None, - "service_tier": "default", - "status": "completed", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", - "usage": { - "input_tokens": 19, - "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, - }, - "user": None, - "moderation": None, - }, - expected_litellm_response={ - "id": "resp_o2at36wcxwspffhlmvydms2hpl24bse46blsmsm52ov4xtt4ordq", - "type": "message", - "role": "assistant", - "model": "bedrock_mantle/openai.gpt-6-luna", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, - }, -) +BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in MESSAGES_RESPONSE.items() if key not in ("container", "diagnostics") +} GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "bedrock_mantle/openai.gpt-6.1-sol", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "bedrock_mantle/openai.gpt-6.1-sol"}, expected_provider_endpoint="/openai/v1/responses", expected_provider_headers={ "authorization": "Bearer synthetic-bedrock-mantle-key", "content-type": "application/json", }, expected_provider_request={ - "model": "openai.gpt-6.1-sol", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "instructions": MESSAGES_REQUEST["system"], }, mock_provider_response={ - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791240611, - "created_at": 1791240611, - "error": None, - "frequency_penalty": 0.0, - "id": "resp_o2fd3mpsaqp564aclj3nmh5776564wp3h5ljwc4dsjgjfncqvonq", - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, - "metadata": {}, - "model": "openai.gpt-6.1-sol", - "object": "response", + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027274, + "created_at": 1792027272, + "id": "resp_n4higs7hpzdjowbay2ujeesbgcxavsl4pbkzhbh7pjpimjjaciaa", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect.", "output": [ { - "content": [{"annotations": [], "logprobs": [], "text": "Hello!", "type": "output_text"}], - "id": "msg_597e5fe9c0fd506dae6deaac23093e3d", - "phase": "final_answer", + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeB+Lw/sASAOaRTS5iOdUyDiwGqyLb6EJDqxBCdFOQvGfAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBaTJyR044dmNxUGt3TndhZkNmeTBpOEFUWlBHSGN4L2s3U1E0UGpyUEp0bUdlZVhXVDREVy9tS2FPdStQUFd6OGc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcJ5O6OH7BzhGiCKEzUwANq0bcBtwE7aXPWBjYzgTyexPQO0PepR8jRQ3zVYRhZpm6K2TOk2lvrp72PZ5N5jG3KGRkWT2Y2jsL9pnkWwE5kN16+aY2OKOY9ZlzQukCAAAQACDHXld1blcGog/Y0exPb0n+8Vo0dT1X9pL401/Nv3/uQlT6BC0wwixOAufgfaY+ev////8AAAABAAAAAAAAAAAAAAABAAABrRdgMGsGAKubmJ7dCj1A9UQgYVFb1O6+NB+Aom8IGUy3Y6E3c63eWNkWXxN8APSjWFwq012VWZ9Q11DzJmGSwM8gtV25ckYezPfNn46VmSj2idcLs2TxTSxP89NAOfSMzWRnW6Dr3lSlx72uQKBrHWYPyAskoHTw4E6CfnFhXotCshAmT5dgImjniqdLFu4CWh2xpQcmYkFbxXakTBsK80TEI8fpJVjGYQbomlrZTwAb8bdyEp0/05xVeMVYC47F0tBY5zFOeZ0Nk0ps2PFteD1HQ5GEt7MRMb+VTGGSbq0wmjY7pvdC0f31cs+0GhEZ1N/3707cGjuMevEMn17YYxJ0oelPOatqdAeSSxBGkEKLmzos0tDcisqwWU/94DL2hmTJWacHamboEN74xZ+KwOjKEoyEr0HOL7n5+B9Oojl7MPdWgBnyGUwCsB3BH8t+h5MwMtMJY451PvO4hgl3qK2cN4WfgoYHRvWM2+rj2iJqYBXFPLo5yqWd3XIIimuTGltinK/0sr1uC/AMQ2Qa0qEB0J4Zxla83H7HCEBg4e8wde2qlXZCG9AxVpPQVd1LijTvn7sPIJgYsmSMwKYAZzBlAjBvywiP8Kq5WqQ+kknoQv+bn4OtwISL+sXhXE+9JK6hhjn02Xd9iSkC/h6IGxAc22UCMQCBBpc3UEvXQSZAxGDy1xUr3JoG4X3sj2Y4yB3SHvBVcxmAWoVo+QAUVMNulgqXbwU=", + "id": "rs_187e1b0bc4995fc9b7fd2796a1889637", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, " + "and non-integer quantities.\n", + "type": "output_text", + }, + ], + "id": "msg_56724f55f0ce5b10a7909c73300f34df", + "phase": "commentary", "role": "assistant", "status": "completed", "type": "message", - } + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":120}', + "call_id": "call_c70cf00600395be0bd35d0db3b1cfa42", + "id": "fc_8bca96735a2c548c9f7777553cf8e5d0", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "in_memory", - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "safety_identifier": None, - "service_tier": "default", - "status": "completed", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 141, + "output_tokens_details": {"reasoning_tokens": 76}, + "total_tokens": 450, }, - "user": None, - "moderation": None, }, expected_litellm_response={ - "id": "resp_o2fd3mpsaqp564aclj3nmh5776564wp3h5ljwc4dsjgjfncqvonq", - "type": "message", - "role": "assistant", + **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, + "id": "resp_n4higs7hpzdjowbay2ujeesbgcxavsl4pbkzhbh7pjpimjjaciaa", "model": "bedrock_mantle/openai.gpt-6.1-sol", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 141}, + "content": [ + { + "type": "text", + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n", + }, + { + "type": "tool_use", + "id": "call_c70cf00600395be0bd35d0db3b1cfa42", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 120}, + }, + ], + }, +) + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "bedrock_mantle/openai.gpt-6-luna"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-6-luna", + "instructions": MESSAGES_REQUEST["system"], + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027280, + "created_at": 1792027279, + "id": "resp_di3jhnf3mehuuht6smddkuygfeo7rzxdxmwvrc52aigdygbts5ca", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect.", + "model": "openai.gpt-6-luna", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJWGzSslRfiCKfPUzG9PfxThM9nB8cV14wc1iKvcbztDAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBb2ZiS1dnQ24waEpudTVSZGtlLyt3emJKY2VIUXlCeU9rWGd6enJDYU5ydXZYMEFmV1NwK01vVEUwZzhHWnpWb0E9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcJhapxvTAKRUzTqbKGUoTlXJqSZjSljHkimXkxwTyexPQO0PepR8jRQ3zVYQcb1ucdG8HiPjq86SsgccbqbIEmYllOUwDws5V0DtxxRnqoVEhkF3h+EG4I5uuB2kCAAAQABDKdaE231TqbLUInCwHGBD9+U9x7lXzuoVQNhj0KATZ6U5+Ki+ZDt6KLjrNb/LM2v////8AAAABAAAAAAAAAAAAAAABAAABdglH8KRxssIDHD4kcMUX0rX0yogE2V0AOj4vmUzgN+tpKEUhA0/BNphpi42nYRZK61EgLzZk42sWn6sJhR5/WKHHn/7N7FppZkt6kFCDTyhRfPimhQT0S7NiACE4511FHXE/WbRGd2/3uuLOx6PfMFlxuKgJHYQuMEttkYRZz1lwzMrtQTyTtghKKz0QaNtp7umhKHlaFVetqVsKqLD7JpKyBxjbe5VojoKmMQEY3dExNYgvCnOBPb8uLhZCcBemLq1ZQzinbMRyRO1bhQmfGsx4GgFbuA4cyJlyzHe+Af4VqcRM94bELBMXiU64UveyqRkWBdocsPfPFpVWa+Hnvl37fgn4jBTnhG1xqNuoVCXX2nV/GAMYHdPwsKweIhxg9I8EcrsXOcTWv26hwDRpmdoHHtJkSqKSbudxEs1F5kr5ELytEFS2OWJwwsMvDfBAlC4boaYtxUBG5sREazNwDNRkhFWHb8MwsXBpOqwOBtu1L2n953WUP2I2tHE6WcZ3DpoD2dP3pwBnMGUCMQDonP31z1HPHjm2mGf7SINbrdFZ5xl5ztMBj3g8hJmIpdWtN/W5dJkc7kfNSErvVrQCMDGAL4pNuVt19O4BfjAmLKnNyz5x/z1FPLGJDz2j+9b0Y0FYBIqafypg88wJ6XSc0Q==", + "id": "rs_8de42923e05d5521b0dbd93c1adcea91", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check the behavior for zero, " + "negative, and non-integer quantities.", + "type": "output_text", + }, + ], + "id": "msg_6a34dc1447f85d8c8a2f9010dd4bee2a", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":120}', + "call_id": "call_316578feac6a57098276bb1f2671b00b", + "id": "fc_c76fa3372e0257699637cc6777775026", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 119, + "output_tokens_details": {"reasoning_tokens": 54}, + "total_tokens": 428, + }, + }, + expected_litellm_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, + "id": "resp_di3jhnf3mehuuht6smddkuygfeo7rzxdxmwvrc52aigdygbts5ca", + "model": "bedrock_mantle/openai.gpt-6-luna", + "usage": {"input_tokens": 309, "output_tokens": 119}, + "content": [ + { + "type": "text", + "text": "I’ll read `shop/validation.py` first to check the behavior for zero, negative, and " + "non-integer quantities.", + }, + { + "type": "tool_use", + "id": "call_316578feac6a57098276bb1f2671b00b", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 120}, + }, + ], + }, +) + +GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "bedrock_mantle/openai.gpt-6-astra"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-6-astra", + "instructions": MESSAGES_REQUEST["system"], + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027287, + "created_at": 1792027285, + "id": "resp_eznficwibpblmnocp4vuvti6d5bxboyxws4vqf5bk2lboyvvbx7q", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect.", + "model": "openai.gpt-6-astra", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeK6rtxWPF9TjmgBuzPaPZhcqwiqUtYeqNCEyD/fgEZAfAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBaUE5eUozT3MzTE1rRFhLWGRuWUFYRjdZU2Q3OStjT3V5LzBIRUNnZXBUWDloQXZncVppTC9qVUc3Y2h3dlQ5QVE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcuTxMV80kCzRfLH6n87syGDT8XBcjqEaE8tDuSQTyexPQO0PepR8jRQ3zVYTdphRKCdq3CJr491CVE1gdVwCTcbqsK/1k7shfoHg93E2fCZXYKq0tnNLGBKhnl90CAAAQAHNHMusCy4BS4gUSPEf8T3DwPC4nfHsYIw/8I2M8N4Z4dxH0a/6ULM8Zaf5qgZInLv////8AAAABAAAAAAAAAAAAAAABAAABmFno3HYlCh8skcb8d7DvpK045XzP/rPTF9J1TXQEj+Oam1ybdDFFN5bbUrMUhvhaAKoUdhp0TPWFBQnQD7rM1QSBtZY60vbMXdscVeDkmAPh7MsZ6OLAY1olCWPsid7OqATdHKm+S6UfnqnvNLJQ0Hp12YYilLOhW5+BLnmqdm04wkU2kvH4G4vxgRK0hFGMGaoKlCIKRvAmwre2DGXdWx85E9jsq2j1ta1XT6/2eCr7zyXn8+tRI69tNwFVRTcaHu8DvSS1LBzdO7ZwNwUr6PbksO44r/sXeNstFU4u3ZQnUhWz8JNym0ydaia57zXQG3/pl8rJaA3IshV+Gm+Ie7N3qySlcIEonqN+cYhQN669G4oBuvCKARc2LFdRB1OXhErskZ8c2c3NA9Xt/dXBbclKY9i/I3Usp46t46mALqz7rA2ft/cGLtX3gNoPPEKL2+i1ZDa0EB1Y7agcESa5pYLKgajhUDbaxupsVQ93wZiCcDTr7SlEfMrBkNFGijc3kpdn1avRFEzVC3H/Ys1cPPYJ93hOUYO42b/RibFpBlkLnKtNf//v81kAZzBlAjAdmUn/WQ5cOGyMLQF2CQxoUrAO9HhKULmwznaURUbjukpBRw8v6PwflSA+mmU8iv0CMQCASBvNjVVmEd4/s4jH27TFub4sikcdi7+LR5HeitoRhyEJIfB41mRciDxhY80hp6s=", + "id": "rs_e6237911ea3e5775acee304296484fab", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, " + "and non-integer quantities.\n", + "type": "output_text", + }, + ], + "id": "msg_f652378c5e52535290a315cff41a7694", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_3986bff59655522d95c62139330145fd", + "id": "fc_aec0dbb1671153029ab817c03715462f", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 138, + "output_tokens_details": {"reasoning_tokens": 73}, + "total_tokens": 447, + }, + }, + expected_litellm_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, + "id": "resp_eznficwibpblmnocp4vuvti6d5bxboyxws4vqf5bk2lboyvvbx7q", + "model": "bedrock_mantle/openai.gpt-6-astra", + "usage": {"input_tokens": 309, "output_tokens": 138}, + "content": [ + { + "type": "text", + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n", + }, + { + "type": "tool_use", + "id": "call_3986bff59655522d95c62139330145fd", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], + }, +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "bedrock_mantle/openai.gpt-5.6-terra"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-5.6-terra", + "instructions": MESSAGES_REQUEST["system"], + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027295, + "created_at": 1792027294, + "id": "resp_fqwklh5v6lnfftfaq6ddfbclshh3vkgmth75hufjjxjrhn726b2a", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect.", + "model": "openai.gpt-5.6-terra", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeCIDnqsrWiO+hx6u0oBst+OnJIA0RIm1eTWC5ORgkFFNAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBMEZqYXRRMkhMSEw1clFlcythQm0wc1Bid2hBWHJhN3ZZOFdGenRwOXlsd0NqSkxBOGxWd2loU2dleHhmTFN0Snc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcWxvahbwUWb5zEZ4mn/Ri1V5Z4KCuYAzvlyiI2ATyexPQO0PepR8jRQ3zVYTwOcBo7nb2ZZfI77C2nwtaEjD275c1LF19dmrCUZ7VPKIYQCMlismkx7f2X1LE4xUCAAAQAPc8v4U/p6fjgu2ZpZgPQUfpPsrIs2TWrckQcNPOhJmhKYDB5gthEp93h1Hf2+ad3P////8AAAABAAAAAAAAAAAAAAABAAABHw7y8mi4YLItPYN81mZNbuQhK0FTG04aQuvQ0966UQxFIMfbeJe+qtyO7uHmyKoz58NNb0ud67KhOPoip6wfSxVcw2RTBCAljj+gkLciZdZlGDiLyutdaHfCRh6StNZDtXG0xPgbn0mN93xnE0hDa05mjcNsNckDaS2KkTMzoXF0tqWoi+Vl0fMSK3CF/49QKyGXYsDeOPB0b8XghpfF1PvMqMI/Ua9Sd98l999mYu71TaBKvgRS9yLWBXg9YcmfZx2hulOUkK2a9aCQ5aN661B0MTdykIm+CokaKogZCv6e/MXncHPPtBeqtuQtsggedinoW/qD09HU+rLUysC+vbHT65YqoVL9wW42XZa79V5QBFYIQL787eVy54qbPxrryWKuTOsN9xRFqaSOv1L10ABnMGUCMFnJHyhVr8ujzWTwnSQCuS+zKrvPgXQfEePxJOR/KFHa7CUXZ8kBuEoW0kVxx3biLgIxAK+8PazVOuepKMy5cuKKsJChOnvQ1Npz7Gpmrx0CXO7X2sIN1WJTlhdNHEclLuwoSQ==", + "id": "rs_a4e6e9898f225c88b4d948b6488bfacc", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to verify the current handling of zero, " + "negative, and non-integer quantities; after that, `tests/test_validation.py` is " + "the next file needed to confirm the expected coverage.", + "type": "output_text", + }, + ], + "id": "msg_a6c79276e3bd594f99585696b01148d2", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_ea239067d1ea5b4f8b96b9240bca0523", + "id": "fc_fb9f552a9d7b5f80917959284de3e9aa", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 109, + "output_tokens_details": {"reasoning_tokens": 23}, + "total_tokens": 418, + }, + }, + expected_litellm_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, + "id": "resp_fqwklh5v6lnfftfaq6ddfbclshh3vkgmth75hufjjxjrhn726b2a", + "model": "bedrock_mantle/openai.gpt-5.6-terra", + "usage": {"input_tokens": 309, "output_tokens": 109}, + "content": [ + { + "type": "text", + "text": "I’ll read `shop/validation.py` first to verify the current handling of zero, " + "negative, and non-integer quantities; after that, `tests/test_validation.py` is the next " + "file needed to confirm the expected coverage.", + }, + { + "type": "tool_use", + "id": "call_ea239067d1ea5b4f8b96b9240bca0523", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], }, ) diff --git a/tests/integration/translation/messages/bases/gemini.py b/tests/integration/translation/messages/bases/gemini.py index b464f9a686e..00d4a7fe5d2 100644 --- a/tests/integration/translation/messages/bases/gemini.py +++ b/tests/integration/translation/messages/bases/gemini.py @@ -1,6 +1,424 @@ +from collections.abc import Mapping from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGES_RESPONSE +from integration.translation.templates.gemini import GEMINI_GENERATE_CONTENT_REQUEST, GEMINI_GENERATE_CONTENT_RESPONSE +from pydantic import JsonValue + +GEMINI_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in MESSAGES_RESPONSE.items() if key not in ("container", "diagnostics") +} + +GEMINI_3_8_FLASH_THOUGHT: Final = ( + "**My Analysis of the Task and Next Steps**\n\n" + "Okay, let's break this down. The user's directive is clear: they want me to understand how the s" + "ystem handles quantities, specifically zero, negative, and non-integer values. The constraints a" + "re equally clear: *I can't modify any files*, I have to explain *which* file I'm reading *before" + "* I read it, and I need to use the messaging client.\n\n" + "So, the first thing I need to do is identify the relevant file that likely contains the quantity" + " validation logic. Based on the user's explicit mention, I'm going to start with `shop/validatio" + "n.py`. This is where I'd expect to find the `validate_quantity` function or related code.\n\n" + "Therefore, my next action is to request the contents of this file using the `read_file` tool. I'" + "ll need to specify the correct path, likely `shop/validation.py`, and I'll include a line range " + "to make sure I get the relevant code (I'll err on the side of caution and specify a larger range" + " until I can see the structure of the file). I'll also ensure that I use the message client to f" + "ulfill the user's requirement.\n\n" + 'My response will follow this structure: "I need to read `shop/validation.py` next to inspect the' + " implementation of `validate_quantity` and see how zero, negative, and non-integer quantities ar" + "e validated.\" Then, I'll initiate the `read_file` call with the path and line range. My focus wi" + "ll be on identifying the precise validation rules implemented in this file.\n\n\n" +) + +GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = ( + "EoUMCoIMAWkUfRNAt6WiTWGx73C1cgarbMkd0STsaBTv0DSrICGKWeyBN9ogaaR8u/oA25JfVEENowEsZIoFwKX85uOsevrt" + "So/sB0HCD+9aN1brlZmpOT413RxVKqvnsk5hBEN6JmhOulxKgpcwftZKsNTnn9NWlvM8DOAXsjiK+4pHx5jNtGQe+T8vd2Wg" + "uqLWu4Gky8NMq+kde0q4jSlC+DYEZ8v5DXO8Tgb+gob/jhGOhvDip7WFY9CPF0X2MI4pbOAAeX2GT+XhteIuYMfEJuA7HAs3" + "IEHTkgqjS/pp6CneKt/BE4jisIalNN1A0xrd+fWd+oQsG6PkOfXpUmQdF//5qVOHKVI8PPIPF3oGH7CD4i7FqBeDnYGCPxrt" + "lTY4DT9nMkOXE6D6wh7hUoJ+Pihckl2JcnJKFIlcnFA5WlG3ph2vgF9iULhhQpRSP3WMu3CTmppGfrZiLRzpi4AztkrwlPRk" + "PZvXd8rFhddD6+o0Oc8Wj3jPHg9FhuAHOQyVGrM6WNLWuTCHQQjvGRfamniStmfOFSOvaBVStHQJ3FTL+y1MVqn+6PDSxjCP" + "1uVmCO0A0bjWxeSEN1pm8Fn8iNIE+oxGbPOPmTOX/I/viZEs7w6vBiIUNna9mMS2pw1asUn3gDCTEl47YtuUa4rMavR7CdsX" + "uaULaPZcYU9LhGr5he1tWdKe1RCvLyUM9Cah8za3ByyQUbqpwE2qmJeQpJc1bmhbMvD3za58oCRWDS6P6EWfJBdyPNhtPhbc" + "gP8w/8uEq1aeoAVcLwJr48bVwWnsUEZ0INRPtgpy55lRayzGo131gtFnt/5lFxz6Q3lopEZlaJgd3X416LngnkS1Yw6uU/J1" + "GPaFlVsYkw/Z/8BkD+Q9pREcYg5Kgq54BOn8fws3TsBtW5g80X8AksR1sCxmwyZkfZx8quis+0gNVmTo3dFtcnW4Q2uk0PUn" + "h8Kv9tTkf0njAZx1ExkgxGLPnS0/7IkpkihF6AuznOl9Hz3+jNjHvscHdx0pFudTFgEa4TcYnQhuj70RJ0NdHC4XgWTUHjeo" + "tlwkunLpmUOavMYLBBrjPcmN8+9czz6d46GGDg2wTqV7jYQT1uyuTOaCF0/y7wWeGeU7gn8KqkgpwcQCXQU2AvqkMCdY4dq/" + "uxh/tlhkzVeMwtTCeMjnMtCMgwarj5VAwivS6e9fc3hbls0qZpJahNkSk4K3+5qABqy0rvcdNEXgMIq02mM5EJab+LJwgY9U" + "BVN67FV5lV9of/p104w3yVPFt7lZlCh3d5xaosZyxjmSt1WlR1b+t/2Wj/08vm2qrpw4lJYa64PwkB6TeQTNiAYaqu6iuVEs" + "/wnZBcAI6Ik3pg4nZ776igMXLgw6zOBKU0uS2gqQcidMNK2mdZGOYWuSjHsMRHztlC+z//Rc9KMuAJ9uEAT+mAH+jEHv+q7G" + "ErL2nMrc4LmViqZ+QKvfpCupH0LvsG/Ab2hThlwAyt/W+vLYtjgNcYnFV5IWVwfuXkYaMkAEZZJf3TO8ja0UZh2VZnuXhbp/" + "rrjaxKQv6CWsXXvbBviBaH3OEKir8m9+SsQVioqheegPSY1bLeDprxrryDhiDUw3bcz/jtCWsGFoTXeN3g0Thdn5DIFfpIq+" + "cQqFOwyyzRpEEYzL78K3VsThS+i7L3dj1WubhgSZ0R6MBtE2BfMWLQYKwU3TOWgiv+GpQGorqXdPN95BVI27+ykf7dVO7NEX" + "S23M4ucImSq6Zd3uDRZaRpHoMSGOTwNcNVNpyrpCIa7kg3MqnBdqjdPDyUDa/cj522XXwzvrj112CGTNOcw3wCbEivSOtT9b" + "Yagt3Zg1uKgKSyQZPQx5qmxVBXBqgo/MViybYA3HYohmtrbwPWXUWzoKNDmjd5rvkzaIfjPGMhYLgKXS5f2gekpAqb6rndYH" + "MgONwdfCmWLr/m+oHuOvlO1tHHcXtrdvbr7TU94/OPYoSFLgDGv7Q2mk1GFdz2xFQYzMH1JdEYtXGwmYXUKIUbVJ6ZQXB750" + "td/Lfz1tZ7G2nPJMMIKl5b123IDjKrY0fZD19tevY54=" +) + +GEMINI_3_8_FLASH_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_8_FLASH_THOUGHT, + "thought": True, + }, + { + "text": ( + "I need to inspect `shop/validation.py` next to see how `validate_quantity` handles zero, negativ" + "e, and non-integer values." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "end_line": 50, + "start_line": 1, + }, + "id": "call_43752", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 64, + "totalTokenCount": 773, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 355, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "5ibQasrzCsbUz7IP89eBkAM", +} + +GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "gemini/gemini-3.8-flash"}, + expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request={ + **GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": {"parts": [{"text": MESSAGES_REQUEST["system"]}]}, + }, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_3_8_FLASH_RESPONSE_FIELDS}, + expected_litellm_response={ + **GEMINI_MESSAGES_RESPONSE, + "id": "5ibQasrzCsbUz7IP89eBkAM", + "model": "gemini/gemini-3.8-flash", + "content": [ + { + "type": "thinking", + "thinking": GEMINI_3_8_FLASH_THOUGHT, + "signature": "", + }, + { + "type": "text", + "text": ( + "I need to inspect `shop/validation.py` next to see how `validate_quantity` handles zero, negativ" + "e, and non-integer values." + ), + }, + { + "type": "tool_use", + "id": "call_43752", + "name": "read_file", + "input": { + "path": "shop/validation.py", + "end_line": 50, + "start_line": 1, + }, + "provider_specific_fields": { + "signature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + }, + }, + ], + "usage": { + "input_tokens": 354, + "output_tokens": 419, + }, + }, +) + +GEMINI_3_1_PRO_PREVIEW_THOUGHT: Final = ( + "**Analyzing Quantity Validation in shop/validation.py**\n\n" + "Okay, so the goal is to get a handle on how quantity validation works, specifically how zero and" + " negative quantities are dealt with in our system. I need to see the implementation. I'm going t" + "o use `default_api:read_file` - that's the obvious choice here.\n\n" + "Now, let's get the parameters right. The file I want is, of course, \"shop/validation.py\". I don'" + "t need the *entire* file, just the relevant section. Let's start at line 1 and go to line 30 for" + " good measure. That should give me enough context, perhaps including the function definition and" + " some surrounding code. I'm expecting to see the `validate_quantity` function in there, probably" + " around line 18 or so, but let's be sure to grab a bit above to get any relevant imports or help" + "er functions.\n\nSo, I'll be reading `shop/validation.py` from line 1 through 30. That should give" + " me the information I need to understand how we're handling quantity validation, especially thos" + 'e pesky zeros and negatives. Now, let\'s run the tool call: `path: "shop/validation.py"`, `start_' + "line: 1`, `end_line: 30`. I'm expecting the output to contain the validation logic.\n\n\n" +) + +GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = ( + "EqsHCqgHAWkUfROgGyzkZ+pi4VHIZ0/S9yiD/m9VZs8L+Rm8Hnfv6gV25Dw2Yc5P0GA0XK9w2l9T1N12/8TW6Jdh15A58C3R" + "hiVDYeyIkhLlk00QjCI3VtdLasRR6T9wieHiXKKIrm/WbfAu5GougrYnUnt9OKFjiH4l8SxQQfXISCA7Fj/+/N0LCzNRug8P" + "DI5X3x3Wug86RcIa9WwNhK1FQ00V2kMzGCdt64Ieg9YqA9sTtgWqRzEc7QbD73tS15WzeSjogaFI1aXbtpTPJCYPrUN3aUqs" + "yXlqhn5sbKJCuDg7M7zOmneh8TbJwTrcpxvHfn8xEHZouCQhY1/WyFGWXvBjzZnJpwoheNkyA7GTOSE1vhB0I5oyS3OwSHn4" + "zLYrcNvwu+2NZp7zc5fImPLxkIOpK2sl8N3q6M01Qnvae4XXtfZ3wNNviM1N72s8I6ZWAYkeS+U+14TZFrL3jsx/Mewi/6be" + "1Y874WiAuq8B3lF3ACnnJ9E8PDTYripxNwhjtveOJC3YCQncxd6CMIf13Xgnj17G9MEdD1HqKnBkLDsfENAfbauaxF+pouX9" + "+JPPHZHyQ91AzcsnXchnFHkZ61goISJILsRikIfzkiVLSopWlsOESzw2ckIA7uxl3WEpON5Qf5/YXEgMIb3OuakYEgLnHdKd" + "WXmmzkel8yc0xxYLzk9uAy2X+ZcKJwbmmU0oJkSOe48zc26xcBZEouznB+LO9q7S9qzh3nN9owa/OAVp/z/5OA+jGCxr6VbI" + "/7bMHcbI1S5oZXN0jjKAlMqNzlo6UWWBB/TPqH2c2PtEYYcNZWhpoQruvBGyGFtT+fj/YrumG1c8EAsBC4NmBecdTMFqfZsH" + "imnQ/qPenuO+qof7iyxZKRhSRTC5qL5PlwuE4tKL4QzwjDDUrOJQ4EapDjYGC7fR143nP52KZGTONQlYtl8HMOh5FT07PExO" + "tHvXMfCh10xkfkWGVAbVWO2mcp5naAAK9eoSeLydygv2P3fZCProx/xPU3RvzDafLGbc2xEO2N67RQnvCIUQ9SwD9iFy4AZR" + "HPaxjxamQLgHW0hDQuRjBzqtXkO5z7y2AKQi1twxv2UAbbZ6ZR605/1ZNbJXHlsf6TCGKZ1/4TXjeQHjLTGf/qb0Jj9ZFxgo" + "0ffs0HxBygsZY7AXvIMVHwgHeyt9nGiz8yPGReQ/1WJEdcmy0GNnYebPuYgWOk9IHAjrpO/zna9cznwDdmzOjmbXlggwqyFI" + "kudjw2Yz" +) + +GEMINI_3_1_PRO_PREVIEW_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + "thought": True, + }, + { + "text": ( + "I will read the `shop/validation.py` file to examine the `validate_quantity` function and unders" + "tand how it handles zero, negative, and non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "start_line": 1, + "path": "shop/validation.py", + "end_line": 30, + }, + "id": "call_29922", + }, + "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 69, + "totalTokenCount": 656, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 233, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "7CbQarXiCMOb6dkPso3s2Ao", +} + +GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "gemini/gemini-3.1-pro-preview"}, + expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request={ + **GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": {"parts": [{"text": MESSAGES_REQUEST["system"]}]}, + }, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_3_1_PRO_PREVIEW_RESPONSE_FIELDS}, + expected_litellm_response={ + **GEMINI_MESSAGES_RESPONSE, + "id": "7CbQarXiCMOb6dkPso3s2Ao", + "model": "gemini/gemini-3.1-pro-preview", + "content": [ + { + "type": "thinking", + "thinking": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + "signature": "", + }, + { + "type": "text", + "text": ( + "I will read the `shop/validation.py` file to examine the `validate_quantity` function and unders" + "tand how it handles zero, negative, and non-integer quantities." + ), + }, + { + "type": "tool_use", + "id": "call_29922", + "name": "read_file", + "input": { + "start_line": 1, + "path": "shop/validation.py", + "end_line": 30, + }, + "provider_specific_fields": { + "signature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + }, + }, + ], + "usage": { + "input_tokens": 354, + "output_tokens": 302, + }, + }, +) + +GEMINI_4_ARGON_THOUGHT: Final = ( + "**My Approach to Addressing Potential Quantity Issues**\n\n" + "Okay, before I make any suggestions, I need to fully understand how this system handles potentia" + "lly problematic quantities like zero, negatives, and non-integers. The instructions are clear: I" + " should examine the relevant files first. Therefore, my immediate next step is to examine the va" + "lidation logic. I'm going to start by reading the `shop/validation.py` file. This is where I ant" + "icipate finding the rules around quantity checks. I need to understand this to determine if the " + "reported issue is, in fact, an issue, and also to understand the context. So, I'll start by read" + "ing that file. I will use the `read_file` tool and specify `shop/validation.py` so that I can ge" + "t the contents.\n\n\n" +) + +GEMINI_4_ARGON_THOUGHT_SIGNATURE: Final = ( + "EukECuYEAWkUfRP1AGBm+VfUg2RBxd+BkfJYxv3HJpn4BHb8bKt8sO2dnYDaU7aro4ErMNkRlXK4cJ/2fewoubknteog7q1r" + "huhmhyZ3Ez8UX5nfeiRktTczwkCT2RL0qY0Yd/qaIWoZcI+QM2XKqHbbHEwW5+MJ9kaQBBzoIz+nyFVfvBmsus9QatIjp2Kk" + "BDXa5HiuVyXobtRz4pgHIJQ0IUb7EBVshaFsa1hwLhKvWBTa9kNOrXWdbcLExvRTzPSyIOC3cxMxwV6UDLmEwrSgqPD+hZ+C" + "YWT9SuQqc1EtbvzNWN2/y4y7ZwekleAEF1Wz+njeqWF01FX4+Der/Cc2lWkoCY9u7jJd4InpLnhKdBUGiSEtTubGMiTVWyAd" + "RKBct6bfstopgnGgqGPM8cW/PCof7+c0gW2nWHU7IDpxB9E5bVb4GIE79+TpWAfVt+qR3nAnk0xN2k6USC8lnf6d9bvhJ+J+" + "Z6loijZBaTOCL8FBMmZFwpwlLyj/9HojuS8DIL8OP1umNF4xqIz8FoGMjUyaeoFX6kHSim3r7kjkfP7h+yZa3seFTqjRLjGu" + "PWrB8abJkP39TNqNaWtGspmAg4Z2PDBSrkW8G29BXkfdbRzV1/1MCqcVed0RE+D3d42/jhzG2j5ul1c0kuZ0RkACrX4WQUWU" + "RhmE4s2z8jaSo6sTmi7nMjKSPV5p2tn9n/yaJoZyDXyXlVMhVTIE5+22YvPEnxWHPQ/aTthfxoi31Mq2fwCrTbkf1OoW/VK/" + "QL903naPf4jpJexUiFDLgylJu+PHEkowuWE0QDC27kgSydvF0wHIcCHHsdU=" +) + +GEMINI_4_ARGON_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_4_ARGON_THOUGHT, + "thought": True, + }, + { + "text": ( + "I will read `shop/validation.py` to examine how `validate_quantity` handles zero, negative, and " + "non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "end_line": 50, + "path": "shop/validation.py", + "start_line": 1, + }, + "id": "call_1649114", + }, + "thoughtSignature": GEMINI_4_ARGON_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 412, + "candidatesTokenCount": 68, + "totalTokenCount": 604, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 412, + }, + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 68, + }, + ], + "thoughtsTokenCount": 124, + }, + "modelVersion": "gemini-4-argon", + "responseId": "8ibQavD9KaOO6dkPs7f0GQ", +} + +GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "gemini/gemini-4-argon"}, + expected_provider_endpoint="/models/gemini-4-argon:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request={ + **GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": {"parts": [{"text": MESSAGES_REQUEST["system"]}]}, + }, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_4_ARGON_RESPONSE_FIELDS}, + expected_litellm_response={ + **GEMINI_MESSAGES_RESPONSE, + "id": "8ibQavD9KaOO6dkPs7f0GQ", + "model": "gemini/gemini-4-argon", + "content": [ + { + "type": "thinking", + "thinking": GEMINI_4_ARGON_THOUGHT, + "signature": "", + }, + { + "type": "text", + "text": ( + "I will read `shop/validation.py` to examine how `validate_quantity` handles zero, negative, and " + "non-integer quantities." + ), + }, + { + "type": "tool_use", + "id": "call_1649114", + "name": "read_file", + "input": { + "end_line": 50, + "path": "shop/validation.py", + "start_line": 1, + }, + "provider_specific_fields": { + "signature": GEMINI_4_ARGON_THOUGHT_SIGNATURE, + }, + }, + ], + "usage": { + "input_tokens": 412, + "output_tokens": 192, + }, + }, +) GEMINI_3_5_FLASH_THOUGHT_SIGNATURE: Final = "ErkECrYEAWkUfRORlQ5j/gkLmyx08FVeIMgg57FJXrjcLwBlt9iuELI6Uc5h+NR/Vm+gEsbdZ5mgFTmKjPF9K/eft8IlXMKm6odJjQOgOYsbt2JHzANo0bpfTmlA6fIi0G2zLbvBVASA6Bdxu1aPQuO4voioQRwm2vomRxH1YbWp8sKXk0DBVbefosldrL0zLJFsi5dFYCtPvw0n9olPVgptHzEdiyXqG+63aPxTooARRQutUH0XWAKR0V+P7qWPt55QKlLaKQqKeBndg9JrEplJihg1sp++y+NAi18fsqXteUS2zIeDtdePGM/GS5oVibwE25zJoziPRdtJhGasFSaA7a3znhW9PF0pBIPAIRPKE4NsQ0FRhpy7ksIXY+0uJ4N+WPPejrtKK6z5x+P0tFkFP0ZNNPM8FZbir1ncVhVxkZS/wWmhc/8TZoRA9ghlTpYhHJ+C4fRVqQqnyRR3SDpVTzB4/sCjBlb434dTH0U3jB4h6V9b/Zx4k4pwUwZTNr2FfgOt2bR7u05DOa+H73OzsNG6zBnMYgBndQdRgk58+l4+UcZdpGKB0lkbHdfD2bminBypEmeJKNRpuc7Smuu0YxcZiY03tzzhHdrUmItqC39OEr2CzRcT9DjFpiWydo3ej9ZkEXeyxoMCckpMmGWh6xbjAnX9gkrPFmUE2rJqblDJWa51i6u/p9Y6ciCq6j4lAy9eBfULIRGQt9pKOXagltOnX0vR0MTDYgWe4dVDSzCI1TV7X/SQYjo=" @@ -55,111 +473,3 @@ GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( "stop_details": None, }, ) - -GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = "EtsDCtgDAWkUfRODYoLRWC+DajYQxOvsLgPh0m8j4NTnd7BflzgPBKFfPW+PU1XsMQuEzviI1qk5mYI0qCfOQNf84PAXXvFA5hMYl+YObaND4G+ZtCdYcolFVfPJQqgK6Kpv20n9hZfLt5JzOS2+HRCLZaokIsZFadN++wqEeEkWQhnKdLGH1lM0fn8Fj/pYq95YLGnB90B8Oaj4qyG6ost2dzRAeAzFSXAko1mD/IgsDrDhEumngCqotdAbPW4jUGYOGDpoXLrBzQZvGa9blRC3ep6NLT0EYMnXImLFoZaLLIBMzVDsmmL0qOg4Gu+uNJlY6cDmtqRgkrcvuvGhh8+lrjUMJVigSsTAoKsTnT3OyCqdNqa+R2aD4WTl1uBFyGY7yXpZ9skQPkV210QNOylZ6exaMA51+W/mohL5j5+OJX7xtVfRIpjp4e0PLkEPxnuviX6OU4ykWZSSiztSXzogbrmnwP7faclRXTXHE5pFlM8y7gDJ9NEwxd5vvIMLdaK2dRWBNtvjI3Cj56sLKygH/j0mQSnt3PonE6Jv6XMJDPbuIzsjzX8t/j3u6WP0bVGnifNNEfIqt6YcKgeWlUTnabRGkWg/fBSyYbiTTEIGhketv57hA9oH7jJZbw==" - -GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "gemini/gemini-3.8-flash", - "max_tokens": 1024, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, - expected_provider_request={ - "contents": [{"parts": [{"text": "Say hello."}], "role": "user"}], - "generationConfig": {"max_output_tokens": 1024}, - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], - "role": "model", - }, - "finishReason": "STOP", - "index": 0, - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 2, - "totalTokenCount": 120, - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10}], - "thoughtsTokenCount": 108, - "serviceTier": "standard", - }, - "modelVersion": "gemini-3.8-flash", - "responseId": "5BjEaunaCtadz7IP-8WKsAk", - }, - expected_litellm_response={ - "id": "5BjEaunaCtadz7IP-8WKsAk", - "type": "message", - "role": "assistant", - "model": "gemini/gemini-3.8-flash", - "stop_sequence": None, - "usage": {"input_tokens": 10, "output_tokens": 110}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, - }, -) - -GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = "Ev8DCvwDAWkUfRMrwgm49bezSdfm90OeW6KeR4kKqExF8s+EorBlW6NWc6XLqdQ2dA5xZ4p0CPNrjuxemR5dt9ch3qMTSgIHTcKKAnxJ0eqsjR61EJWFP4JNdigiymymc7UNs/zLZer+qPH2XQLD9r85O3NVBeupYk6xy6395CZYygF9oVJD3WNXwlefvThnvH/3rDsnO0FBfcrvxRHiSVTD1Moe+uTVV2w3vKKSCxUb64w5lquEjFx+AO/jiJIc3McPvAOvUr0I/2fMCWLcO7Y5sV6zuN8qpaQKitC/Ev09cl3SAbKzwjO3gBgYVmF5PcPY5HT8S3bwCez2aOgX7BCN+FItlZ4wMsZStLIY38XMQtbibVRHmiufN86IMkoD8Yxb1lnK7aaS+anSPkn4M4zkkiwAUdjt14k6JvMBk9J4duvTHLP0BiSaLLkTe6Ufj9cRMxZKCP0ew4DiLMwejuQoSpC4aLP1gmli4eOUcyq/g2/o0kByY8Fl2vb54eiXADp4fhhIEAZFe4J/0x29ZZVKx8KEInDahP7tslzwah3PWfX/K1jXzqX93mo4a/0Ec1bM/sqwiEWSHNhrPpyXcTkmWQ1Bot+PTARnzAjeXvoTR5sajbNKj1SdCC0RY7zb8OqGZHDQxgNw7ghTjYMTdBJsdCMQfnl3dvqmw6zM/aplZg==" - -GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "gemini/gemini-3.1-pro-preview", - "max_tokens": 1024, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", - expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, - expected_provider_request={ - "contents": [{"parts": [{"text": "Say hello."}], "role": "user"}], - "generationConfig": {"max_output_tokens": 1024}, - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], - "role": "model", - }, - "finishReason": "STOP", - "index": 0, - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 2, - "totalTokenCount": 115, - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10}], - "thoughtsTokenCount": 103, - "serviceTier": "standard", - }, - "modelVersion": "gemini-3.1-pro-preview", - "responseId": "5RjEaqfwC7XYqtsPp6TuoAY", - }, - expected_litellm_response={ - "id": "5RjEaqfwC7XYqtsPp6TuoAY", - "type": "message", - "role": "assistant", - "model": "gemini/gemini-3.1-pro-preview", - "stop_sequence": None, - "usage": {"input_tokens": 10, "output_tokens": 105}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, - }, -) diff --git a/tests/integration/translation/messages/bases/openai.py b/tests/integration/translation/messages/bases/openai.py index 87fbd35cd8e..f53baeafec4 100644 --- a/tests/integration/translation/messages/bases/openai.py +++ b/tests/integration/translation/messages/bases/openai.py @@ -1,7 +1,8 @@ from typing import Final from integration.translation.case import TranslationTestCase - +from integration.translation.messages.templates import MESSAGES_BRIDGE_RESPONSE, MESSAGES_REQUEST +from integration.translation.templates.openai import OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, OPENAI_RESPONSES_RESPONSE GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -290,289 +291,496 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) - GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "openai/gpt-6-luna", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "openai/gpt-6-luna"}, expected_provider_endpoint="/responses", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, - expected_provider_request={ - "model": "gpt-6-luna", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "include": ["reasoning.encrypted_content"], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, "model": "gpt-6-luna"}, mock_provider_response={ - "id": "resp_0dda38229cbb1547006ac3f0195a8487d0818613a635a9e0e2", - "object": "response", - "created_at": 1791225881, - "status": "completed", - "access_programs": {"cyber": "daybreak_blue"}, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791225882, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6-luna", - "moderation": None, + "id": "resp_03ec491c5caae029006ad0239c083487d099a21d60be6b5fde", + "created_at": 1792025500, + "access_programs": {"cyber": "daybreak_blue"}, + "completed_at": 1792025503, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "messages client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect." + ), "output": [ { - "id": "msg_0dda38229cbb1547006ac3f019f55887d08644c44c51109378", + "id": "rs_03ec491c5caae029006ad0239c80c487d0b425db4d74878ab3", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COfZvmrZboi5Ffbs6U_yMF1jaQmGVq5BnqydEOn98Tt_afVrM4H653kYvwFlbyesZJUOa4MLSEHMVc5ZNHlI88WpsGB2C0cdOmXO-S8Xz2wv2TYaf-d7ux1ejKPGkbop3sk2yS2XxQahiybVv5r_opD4YkEVxPoKhnVTfNF0bcglJj1JQbXYSE5qNyAVFeO6kkYN_sRP628gc_-txGgIgJbLn-uTsZ_3aQZ8TmejJJaiXLcauCUyCnYHtw0BncdwxaRiDsAUhrL_HuwLgGc3KHP0mc_IjapNLhVAcqeihSO4rmidsYX6Y7bjb4yhqPqE0-NnPZRO8BOcpIb-qU89htZHrkaSA_MrZj2FD4fk8_btQHTF9C9hM-UjRUJxqMtyuOyNMZdWjBgErNoCzslro-Sty8pWlIYRjzOlQQHZLBrC3EuKvOTEBNNjWXeRCGwZTZR6C5SyE0mj11YCwi67QeYfp8ujvrziJq1Bt7u78vnlPY8bMd0AUz0SURgP2jnlreu11EMWZuR-u6ZMLqyu1HeunBMs82y-5ef37g3evBV7Ym6AjwsiP0SqTM3KeaY5HMkgaZi9G2gwO2g8ZkLsJpy-Bk01KwNYD7QXAxrUXziEAzby_AqAOOZ9_w5mkZSSoJuelWs6nB4stKCaj8lXIanf58DkNjze2Ocd9irMxIz4HUzXcYWyjDa1-0isrR_gQiOWOaCzuIFWPcVrin4Mli-niDnPYcTnNkP4ab1jmTztYb74x2seE9d5voFc7suPLzmqGuFya8fx42A-i7EP-bXM1D5_fWiTAGWbappILLOJzs3ohcO0jBe1NkkxrfVMIJD5m7CVzmOQ5ydhyWHgSNRueFPE2Bz6wqmfnU7bzrAK8DOH1NiA1LMogV7iGBlu4egfKgs7B1m7u1x1YCG5OcBHbm_67VkxxhO95IfwYYEMawg-uxXYUbAx-L_vKnIpyKq1pSaKm-hsvGMm8MvxSviOD2xkxBHElb1rZ6tkWRdR60tSRELtad5F5ItHHZe27tq-XMFZAB2UwxtvwwTZ5wD2C5PmxREMk7XF_YFR8vdVdEqfaLelx6NoeARXQ5Hm7kdiPT33RAVtyvaXuOgAPt4afpmSb0_iW9VpyU7HnHStXoKdw8gWjW-BnjJ7ZDsqy-KFTRRlj8ikcR0pAFafz-GXmiyn7EVc7rfTFYNN7c91SADAhBg2Gtk1c5EGVLCa9uocAYan6MTkzb-bOorPCI-zE-rztEp9L2Lm1TfyUNEFaIRHuG7GAwIsKvnXBwJlwlJA5McCehhqc1V6-1kqcGFSenXBvzRdXr4Wh-hAqT5gZyG0bRQIVjAiITYCbeQtahw3ioGkxUwoTq0XRZPvy0_B71b76QSbEpnVYL0b5oaBia-4qy7tJpaRkhS6pDY3bjxgWt6yPzk6sTWr_Gbt-nMgxCpkHFYI_KMSI4XvOOcPkNGgVK7X278xisXj3cx72rSaBhpRlNKkEtqsIxDTHOWNk9NZooClcTlTrHPPjo0D_3nLMdw94z5V-x0qsRsk3YrUzfIMUPeBV5ctPo5shpr4ycmLPhrnbT5U4FeaZ27AdgFDuhAeXb6AYPJ6VfOVcyZUibLQ50l2XtxjTP0uiM3fMt_7TiyYjQL_3cBv-vQRViXQyI7_7glU0b9EK-SEfl_6f9NRqWiLabP_bb0ZIrGItQ2DxEJpUmhRAErRlq0a5rlnt-We644LT-i8dvBybsnbbkwV8mqRQrAWCFHrCzPALskgmzRpuUaVBXtBhPKkUKnpwav-TseWr-Pog2iLNNfBD37EAQMPsnslLhuNeZ7-bnNJCerPiiCuNJaPd2Iy71quHCOznW7PDajSa4XvFv3GPeT_WluUD_hPRnDxFR0pFX266UJcNcNmP62ogUtfIByf5dyuwvQAFBk0Tplox1U08fKo8KabYWYVzEYQ9XiLXWgq1OSLdc9teuPP5_aFTL2VjvLgIJ8E8H8xhbk9qnPFvng0e-twGn_rKbEqIceWJ7wVqJSTMAqjJy8nBDelgPWCJVVXQ3NAgGlcAUHO0wv-9YGagfklvXrDfMrjKF8Y9YVqmg8a6FFFmCykZSmiCl-MwjHhxO9c1SlT4pDN3c5dt-V9lYGkGv-K1j392wPZOxnoH92AhX1k-lJirRrgMSj6TgQuUQynaxTL01zwD1ilNuAa6966U-KP7V5peoq-4oENlEhhRdW8HjoHl1jQlP00_7K2f5TFmzKyNPHxq7londzlTpautQp59102404xO7CRjTq4hncweSopzlR0CsSQZmy9ZDwbfOIQczBwY1lwfLvlEflZAzRgDPu-CcXTLivLNmHlXKKBKHQd35sxO3KDmgscSZuO61DvEE3SvWK_XYA46vSSHa4OSWZG1V7IDJHddPQJ65kWHbHfxCVhPY_yH52_jtgjOTUBRt_3mMFizWGenQY-tpWoe7Fy-Ex9vzXGDPi-D9QlmluqCZ1Oj0hv5LJbv65AAh5sLSR2lb_Gs1lEI8_ZMoypadUWUTAdiIQqx4nEDn6Yf9oSWVGxiPxgPlFcn_Upkf0MZqookTf7ZVlpgon2lA5MRP9O3-V9EdU8WsAfw2VV3UlygpAOHs0UQA0C691CY3PHjckcRonszezCd2JPmPlIdm-gh2aubk2IUE9NMaVdgRtc-8ysv7GHnGCvmp8FKDS5f4XytuXO_hEOOn5T8Ug-iENW4WOkIJ0jbtmz15NITlrVbPAnGhMSW0y6BfsSn-b9kloFDCA-Mbb4S-IMY8v80M4OkuhV83WPHsXl3yFEiokO02MphzPvhjoixb4OhnsTKG9vUyw4E4wJgGlxGSczNdkBcqiRISJviiRAxnpPQ4XEQ-RkuOjs1AB_cegluyWezH5LovF4S598IJYOAi1oL6Txe0gw4kJVhBmG2hKBscezt30Deu26Qx3Etk9cN3XMirfot7dBoyNZS1Jj2fpsoMbPpxwLhsIIWbWUuN2nKDqDrK199AzGD8-lki0LZh0QOxBcqbtgDDMWp_BpUtutFWL6TZm8DObtIbl4fGy12y1t4kaN-E9mLpzXQXOVjrUbP_tQlNZk7p5YOTeV_NnpunzpNpAbHXXQIukQ1xOlGvHFZ6CRFC-Yknx72aizrZf6aZH_gy9pWKmOhdqWFQ2PJLqGVLIo3mS4yoFC04AivGXdAjdrZB7odufujij9dspXWBO4Tv-dzL29O-j6p-eT6y0HuKachlFpLKpJxzsjIU1DjX0P_0pSVhemfpZDh_nV9eC2w7XNm6logVPIolXiQeypd70LOKnb-wGwxRNlcus6BUSF_BIlplV1S0TTne5PgYU3OCuOPJAwSEW0Q0lkm3lTU8nLyiQd25bV5UeD2wPvxACIPM7_cUyoGkSQ96Ip-NM607-e912AEVR93ZIVK-UHcu8MxSwWCykNaokgs4mEVK5QkY=", + "summary": [], + }, + { + "id": "msg_03ec491c5caae029006ad0239e411c87d0bfb3c2c04bc6cd9c", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I need `shop/validation.py` next to see how it treats zero, negative, and " + "non-integer quantities.", + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_03ec491c5caae029006ad0239e978c87d0a3ad2d7a8026ad6f", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":120}', + "call_id": "call_6JuM80yE9acKhYmgxQKWll9Z", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 214, + "output_tokens_details": {"reasoning_tokens": 150}, + "total_tokens": 523, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ - "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo0ZmY4OWU3OTgyMGFkZDVjMmM1ZmZlM2MyODk4ZDNkOGEzZDczY2IxY2ExN2JmZGMxYTRmNTRlZmFlNzhkMTIzO3Jlc3BvbnNlX2lkOnJlc3BfMGRkYTM4MjI5Y2JiMTU0NzAwNmFjM2YwMTk1YTg0ODdkMDgxODYxM2E2MzVhOWUwZTI=", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo0ZmY4OWU3OTgyMGFkZDVjMmM1ZmZlM2MyODk4ZDNkOGEzZDczY2IxY2ExN2JmZGMxYTRmNTRlZmFlNzhkMTIzO3Jlc3BvbnNlX2lkOnJlc3BfMDNlYzQ5MWM1Y2FhZTAyOTAwNmFkMDIzOWMwODM0ODdkMDk5YTIxZDYwYmU2YjVmZGU=", "model": "openai/gpt-6-luna", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 214}, + "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:gAAAAABq0COfZvmrZboi5Ffbs6U_yMF1jaQmGVq5BnqydEOn98Tt_afVrM4H653kYvwFlbyesZJUOa4MLSEHMVc5ZNHlI88WpsGB2C0cdOmXO-S8Xz2wv2TYaf-d7ux1ejKPGkbop3sk2yS2XxQahiybVv5r_opD4YkEVxPoKhnVTfNF0bcglJj1JQbXYSE5qNyAVFeO6kkYN_sRP628gc_-txGgIgJbLn-uTsZ_3aQZ8TmejJJaiXLcauCUyCnYHtw0BncdwxaRiDsAUhrL_HuwLgGc3KHP0mc_IjapNLhVAcqeihSO4rmidsYX6Y7bjb4yhqPqE0-NnPZRO8BOcpIb-qU89htZHrkaSA_MrZj2FD4fk8_btQHTF9C9hM-UjRUJxqMtyuOyNMZdWjBgErNoCzslro-Sty8pWlIYRjzOlQQHZLBrC3EuKvOTEBNNjWXeRCGwZTZR6C5SyE0mj11YCwi67QeYfp8ujvrziJq1Bt7u78vnlPY8bMd0AUz0SURgP2jnlreu11EMWZuR-u6ZMLqyu1HeunBMs82y-5ef37g3evBV7Ym6AjwsiP0SqTM3KeaY5HMkgaZi9G2gwO2g8ZkLsJpy-Bk01KwNYD7QXAxrUXziEAzby_AqAOOZ9_w5mkZSSoJuelWs6nB4stKCaj8lXIanf58DkNjze2Ocd9irMxIz4HUzXcYWyjDa1-0isrR_gQiOWOaCzuIFWPcVrin4Mli-niDnPYcTnNkP4ab1jmTztYb74x2seE9d5voFc7suPLzmqGuFya8fx42A-i7EP-bXM1D5_fWiTAGWbappILLOJzs3ohcO0jBe1NkkxrfVMIJD5m7CVzmOQ5ydhyWHgSNRueFPE2Bz6wqmfnU7bzrAK8DOH1NiA1LMogV7iGBlu4egfKgs7B1m7u1x1YCG5OcBHbm_67VkxxhO95IfwYYEMawg-uxXYUbAx-L_vKnIpyKq1pSaKm-hsvGMm8MvxSviOD2xkxBHElb1rZ6tkWRdR60tSRELtad5F5ItHHZe27tq-XMFZAB2UwxtvwwTZ5wD2C5PmxREMk7XF_YFR8vdVdEqfaLelx6NoeARXQ5Hm7kdiPT33RAVtyvaXuOgAPt4afpmSb0_iW9VpyU7HnHStXoKdw8gWjW-BnjJ7ZDsqy-KFTRRlj8ikcR0pAFafz-GXmiyn7EVc7rfTFYNN7c91SADAhBg2Gtk1c5EGVLCa9uocAYan6MTkzb-bOorPCI-zE-rztEp9L2Lm1TfyUNEFaIRHuG7GAwIsKvnXBwJlwlJA5McCehhqc1V6-1kqcGFSenXBvzRdXr4Wh-hAqT5gZyG0bRQIVjAiITYCbeQtahw3ioGkxUwoTq0XRZPvy0_B71b76QSbEpnVYL0b5oaBia-4qy7tJpaRkhS6pDY3bjxgWt6yPzk6sTWr_Gbt-nMgxCpkHFYI_KMSI4XvOOcPkNGgVK7X278xisXj3cx72rSaBhpRlNKkEtqsIxDTHOWNk9NZooClcTlTrHPPjo0D_3nLMdw94z5V-x0qsRsk3YrUzfIMUPeBV5ctPo5shpr4ycmLPhrnbT5U4FeaZ27AdgFDuhAeXb6AYPJ6VfOVcyZUibLQ50l2XtxjTP0uiM3fMt_7TiyYjQL_3cBv-vQRViXQyI7_7glU0b9EK-SEfl_6f9NRqWiLabP_bb0ZIrGItQ2DxEJpUmhRAErRlq0a5rlnt-We644LT-i8dvBybsnbbkwV8mqRQrAWCFHrCzPALskgmzRpuUaVBXtBhPKkUKnpwav-TseWr-Pog2iLNNfBD37EAQMPsnslLhuNeZ7-bnNJCerPiiCuNJaPd2Iy71quHCOznW7PDajSa4XvFv3GPeT_WluUD_hPRnDxFR0pFX266UJcNcNmP62ogUtfIByf5dyuwvQAFBk0Tplox1U08fKo8KabYWYVzEYQ9XiLXWgq1OSLdc9teuPP5_aFTL2VjvLgIJ8E8H8xhbk9qnPFvng0e-twGn_rKbEqIceWJ7wVqJSTMAqjJy8nBDelgPWCJVVXQ3NAgGlcAUHO0wv-9YGagfklvXrDfMrjKF8Y9YVqmg8a6FFFmCykZSmiCl-MwjHhxO9c1SlT4pDN3c5dt-V9lYGkGv-K1j392wPZOxnoH92AhX1k-lJirRrgMSj6TgQuUQynaxTL01zwD1ilNuAa6966U-KP7V5peoq-4oENlEhhRdW8HjoHl1jQlP00_7K2f5TFmzKyNPHxq7londzlTpautQp59102404xO7CRjTq4hncweSopzlR0CsSQZmy9ZDwbfOIQczBwY1lwfLvlEflZAzRgDPu-CcXTLivLNmHlXKKBKHQd35sxO3KDmgscSZuO61DvEE3SvWK_XYA46vSSHa4OSWZG1V7IDJHddPQJ65kWHbHfxCVhPY_yH52_jtgjOTUBRt_3mMFizWGenQY-tpWoe7Fy-Ex9vzXGDPi-D9QlmluqCZ1Oj0hv5LJbv65AAh5sLSR2lb_Gs1lEI8_ZMoypadUWUTAdiIQqx4nEDn6Yf9oSWVGxiPxgPlFcn_Upkf0MZqookTf7ZVlpgon2lA5MRP9O3-V9EdU8WsAfw2VV3UlygpAOHs0UQA0C691CY3PHjckcRonszezCd2JPmPlIdm-gh2aubk2IUE9NMaVdgRtc-8ysv7GHnGCvmp8FKDS5f4XytuXO_hEOOn5T8Ug-iENW4WOkIJ0jbtmz15NITlrVbPAnGhMSW0y6BfsSn-b9kloFDCA-Mbb4S-IMY8v80M4OkuhV83WPHsXl3yFEiokO02MphzPvhjoixb4OhnsTKG9vUyw4E4wJgGlxGSczNdkBcqiRISJviiRAxnpPQ4XEQ-RkuOjs1AB_cegluyWezH5LovF4S598IJYOAi1oL6Txe0gw4kJVhBmG2hKBscezt30Deu26Qx3Etk9cN3XMirfot7dBoyNZS1Jj2fpsoMbPpxwLhsIIWbWUuN2nKDqDrK199AzGD8-lki0LZh0QOxBcqbtgDDMWp_BpUtutFWL6TZm8DObtIbl4fGy12y1t4kaN-E9mLpzXQXOVjrUbP_tQlNZk7p5YOTeV_NnpunzpNpAbHXXQIukQ1xOlGvHFZ6CRFC-Yknx72aizrZf6aZH_gy9pWKmOhdqWFQ2PJLqGVLIo3mS4yoFC04AivGXdAjdrZB7odufujij9dspXWBO4Tv-dzL29O-j6p-eT6y0HuKachlFpLKpJxzsjIU1DjX0P_0pSVhemfpZDh_nV9eC2w7XNm6logVPIolXiQeypd70LOKnb-wGwxRNlcus6BUSF_BIlplV1S0TTne5PgYU3OCuOPJAwSEW0Q0lkm3lTU8nLyiQd25bV5UeD2wPvxACIPM7_cUyoGkSQ96Ip-NM607-e912AEVR93ZIVK-UHcu8MxSwWCykNaokgs4mEVK5QkY=", + }, + { + "type": "text", + "text": "I need `shop/validation.py` next to see how it treats zero, negative, and " + "non-integer quantities.", + }, + { + "type": "tool_use", + "id": "call_6JuM80yE9acKhYmgxQKWll9Z", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 120}, + }, + ], }, ) - GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "openai/gpt-6.1-sol", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "openai/gpt-6.1-sol"}, expected_provider_endpoint="/responses", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, - expected_provider_request={ - "model": "gpt-6.1-sol", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "include": ["reasoning.encrypted_content"], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, "model": "gpt-6.1-sol"}, mock_provider_response={ - "id": "resp_03474a48789eb520006ac3f01d28ec87d092a151bed4f1cde6", - "object": "response", - "created_at": 1791225885, - "status": "completed", - "access_programs": None, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791225886, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6.1-sol", - "moderation": None, + "id": "resp_0af67cb78bead2c6006ad023a7273487d0b1768b77f6956b73", + "created_at": 1792025511, + "completed_at": 1792025514, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "messages client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect." + ), "output": [ { - "id": "msg_03474a48789eb520006ac3f01e31e887d0a7384c283c8d0934", + "id": "rs_0af67cb78bead2c6006ad023a85d5c87d0810de0f56bf80208", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COqmCc2HPj-8NSt1bzfstH4-YXiTtNvZLyVMciprSqFmNbPXwcPXWrps8lsryqoAY5MuRQ_pO9yNIsCVBRlHXitSdObtJRB5Ld7TGQnbUekIsl2TNIFPsnnc8PYSdVrJlcG3FZmykyeSVHblM5F9SaPzHFzOfvCdYQtN52yt2lPhjYailjRidMwkl4KDGQ1BlcR3U29KKxo0020AqMMvsAht5kx5aAFaKfaIum5jctQrjgL8Y99h6sV3gc3LwI0W6ZR2IIqs6e6ADrVIGdVmSpn2GZTHmz6vUFaROU2-UTbVL7QrCy0BUg5CEqjnZ563lqsEYuWOxTVK0RXCPonWwmhfSllKffjLxtGuaq4dtl6v3lGIt-BW3-gJ1WqhtfTDV5VWTAliJhx09kgGhXz7VuxQRQMBSy--M9nBhdNJ1KfkbC0YDUcq5LT_cKh-KZU_-jNWCSQa7hvNJP-aCQ85oL-av-9fT5Wb5ovLUzELiBDooQLH8SalXQgFo-GX_wvSF-IH7byXwCka0F9fyIW6FjmRkirX8h_WF7pLDJeZDO5_8BWbxGOxDI6hEURHW9HLGG-e8S_lp1cVc7i37r-3KNqOwVlnlxEn2hj9TmDvTxRk4yv71AoSuwF8j3Wsh-im1HgmsNf9sdKZIZUTJTdvGCNuf54xXedUEdfoInFspUuiLXj_5KwWpckwJD0y1Fs8oEfTq2Y0Bn3LdBvwrAGgijcwnDsj6nRa2L9oMLrbHROG3xWIuTC6JyKIlVo8o7cGsM6XdxV6P9f3Wc-UgScNzc-fleF3oWzVq2gENtufLiwn9y3z1k_Hb62pbgIV6dmhLepqVH7WAyHajYt9GyxHB_FZbni-_rujbci3-3Jcj451W8PFg0TjhlyUzETBQwv6y6-zORZcbD7TqKe6XnUUGxHEMbPKBCmiIxOIUuOHpNOEjmdQpgOuPR16k_j0YjoA1gJJovIqwB1g6w44nebjzzsky0uRWmIhqeNaIFPPtIUDr05z1ocxgFeHJxum5YAh4d46vUY18dpffRyghwrd1rvIxk2EWW2rmruR4nEDTT405t3RX4kjnhGHuyShSLM_jinQm5LcpLDXUWgRnilpB1RVB8eZUSNYePAhgDRbacr8IC4AvQ7X8xUtcglbDJmMYQZQRFrkGfVoMUcyAkEJamLjccYDFrO5rxUFfyUHmGVxFUY85NW9XCwAFOx7h3606gjN9KVKQPMAg-dUnXbfhc4PCY3iRaS0djErOEugRN6f7iniy8td2YRI2WhPD5O1-qsvTDfiIhymTFq2L0v4f3UEeIa2wnfq4EdnAgla4D14w7T1D1Wg4SiP-dnF0_vKy8md5vlQjpbPGbYJkzB9DBNwJ8qR-7s1jCtZj2fViHuIGwTReI3OXUoeao4FHC8pndoAZbEpbTmax7BEYw7JqlUhIXWF2x8NzeYr-sjBbo1kbzKbtnidxOwrngjU428TzTPEjr9WA9VpmxXNoSOPL7I_qsN8ulnGegsnxQOEIarRK9CE7C8AwtMRMi8lClAFaMwMAoD8jWMNCSEa6mtyv08AikNCt3ij6cW2JszTRO3Qu4lRTsSnta41Qyk16Zc4xEPOs58PhY90t_atHYUWRpydUerE2nvRnPF9eTbQvSe8-iZ0BqkXqdlnVAAIzwOIfVZRua0p8eU3iPlzKteg9F02Jb5HkOsvi2Y4M-ZS5i0Tg4iOeuKHkeiFdZs6EMytIwz6lPGKLhYii6HW0Oe0KU_WQrIVNdXMcs5EWZGqMIdEhf09ZpO3ONcWIYswjgng6W8WMqwplHyHMcjnhzaVbIhs6rsVgTwGJawtbmD5q6aYnJCRq4zI68wzKOe7d6av8jwMGZS-dd8cdHj_XyShws1cWLhLJCaGW_vHKA0i-D1WTDeKSev31IQKCXBSRMB5URPkHu6dNDm1UyDiLGlJBx6iuKTizGeGhb4o2SQ1y1ISKyQRhKuNih0XgY40hn4x-6Scqu6_thPey-4F32BeTnbzNLGQ0wXE--dcIVRbAZMEB7wn7V4mRczG4gTG9ZnqarfZnHMPVUTZ0AxxZ2d_ifef1wQv9aRyKNRVLiBxvSqJtu7NnDMZT_U8mXCc_OIREi9wn-6esDbmxYYeBpNPR6Oq0fbsBbr--A_nVUz99MAAvkSnLhV3zdAHmMgjqNJNCmr2kPtGyle-GqdBx0LYQh4mtc5FdqLZUz9dz6gv_ERua952na3S1UnJr__8zVwofLEPGjiC5_g1nQuVCRE3QVSufMmw4gQL5kdUSuZGEm3RN__6DwAIS_X-E-aBv3utOMs92JWGandVEze8K7p4wQr3gSL5zlepjWHeYPZjiXCnGY=", + "summary": [], + }, + { + "id": "msg_0af67cb78bead2c6006ad023a9b1e487d09ad5bb0d5cc96bde", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.", + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_0af67cb78bead2c6006ad023aa22c487d0b76512608a221ae3", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_H6dablHqZFScV63Y29mQXJvf", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 140, + "output_tokens_details": {"reasoning_tokens": 75}, + "total_tokens": 449, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ - "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDphMTYxN2NmYzNlODRjNzYxNDdjNTgxMmFhMjIwNGU0MDczOTM0M2YyMTg4YzQxNzIxMzRlOTc1ZTQ0YzQ3MjgyO3Jlc3BvbnNlX2lkOnJlc3BfMDM0NzRhNDg3ODllYjUyMDAwNmFjM2YwMWQyOGVjODdkMDkyYTE1MWJlZDRmMWNkZTY=", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDphMTYxN2NmYzNlODRjNzYxNDdjNTgxMmFhMjIwNGU0MDczOTM0M2YyMTg4YzQxNzIxMzRlOTc1ZTQ0YzQ3MjgyO3Jlc3BvbnNlX2lkOnJlc3BfMGFmNjdjYjc4YmVhZDJjNjAwNmFkMDIzYTcyNzM0ODdkMGIxNzY4Yjc3ZjY5NTZiNzM=", "model": "openai/gpt-6.1-sol", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 140}, + "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:gAAAAABq0COqmCc2HPj-8NSt1bzfstH4-YXiTtNvZLyVMciprSqFmNbPXwcPXWrps8lsryqoAY5MuRQ_pO9yNIsCVBRlHXitSdObtJRB5Ld7TGQnbUekIsl2TNIFPsnnc8PYSdVrJlcG3FZmykyeSVHblM5F9SaPzHFzOfvCdYQtN52yt2lPhjYailjRidMwkl4KDGQ1BlcR3U29KKxo0020AqMMvsAht5kx5aAFaKfaIum5jctQrjgL8Y99h6sV3gc3LwI0W6ZR2IIqs6e6ADrVIGdVmSpn2GZTHmz6vUFaROU2-UTbVL7QrCy0BUg5CEqjnZ563lqsEYuWOxTVK0RXCPonWwmhfSllKffjLxtGuaq4dtl6v3lGIt-BW3-gJ1WqhtfTDV5VWTAliJhx09kgGhXz7VuxQRQMBSy--M9nBhdNJ1KfkbC0YDUcq5LT_cKh-KZU_-jNWCSQa7hvNJP-aCQ85oL-av-9fT5Wb5ovLUzELiBDooQLH8SalXQgFo-GX_wvSF-IH7byXwCka0F9fyIW6FjmRkirX8h_WF7pLDJeZDO5_8BWbxGOxDI6hEURHW9HLGG-e8S_lp1cVc7i37r-3KNqOwVlnlxEn2hj9TmDvTxRk4yv71AoSuwF8j3Wsh-im1HgmsNf9sdKZIZUTJTdvGCNuf54xXedUEdfoInFspUuiLXj_5KwWpckwJD0y1Fs8oEfTq2Y0Bn3LdBvwrAGgijcwnDsj6nRa2L9oMLrbHROG3xWIuTC6JyKIlVo8o7cGsM6XdxV6P9f3Wc-UgScNzc-fleF3oWzVq2gENtufLiwn9y3z1k_Hb62pbgIV6dmhLepqVH7WAyHajYt9GyxHB_FZbni-_rujbci3-3Jcj451W8PFg0TjhlyUzETBQwv6y6-zORZcbD7TqKe6XnUUGxHEMbPKBCmiIxOIUuOHpNOEjmdQpgOuPR16k_j0YjoA1gJJovIqwB1g6w44nebjzzsky0uRWmIhqeNaIFPPtIUDr05z1ocxgFeHJxum5YAh4d46vUY18dpffRyghwrd1rvIxk2EWW2rmruR4nEDTT405t3RX4kjnhGHuyShSLM_jinQm5LcpLDXUWgRnilpB1RVB8eZUSNYePAhgDRbacr8IC4AvQ7X8xUtcglbDJmMYQZQRFrkGfVoMUcyAkEJamLjccYDFrO5rxUFfyUHmGVxFUY85NW9XCwAFOx7h3606gjN9KVKQPMAg-dUnXbfhc4PCY3iRaS0djErOEugRN6f7iniy8td2YRI2WhPD5O1-qsvTDfiIhymTFq2L0v4f3UEeIa2wnfq4EdnAgla4D14w7T1D1Wg4SiP-dnF0_vKy8md5vlQjpbPGbYJkzB9DBNwJ8qR-7s1jCtZj2fViHuIGwTReI3OXUoeao4FHC8pndoAZbEpbTmax7BEYw7JqlUhIXWF2x8NzeYr-sjBbo1kbzKbtnidxOwrngjU428TzTPEjr9WA9VpmxXNoSOPL7I_qsN8ulnGegsnxQOEIarRK9CE7C8AwtMRMi8lClAFaMwMAoD8jWMNCSEa6mtyv08AikNCt3ij6cW2JszTRO3Qu4lRTsSnta41Qyk16Zc4xEPOs58PhY90t_atHYUWRpydUerE2nvRnPF9eTbQvSe8-iZ0BqkXqdlnVAAIzwOIfVZRua0p8eU3iPlzKteg9F02Jb5HkOsvi2Y4M-ZS5i0Tg4iOeuKHkeiFdZs6EMytIwz6lPGKLhYii6HW0Oe0KU_WQrIVNdXMcs5EWZGqMIdEhf09ZpO3ONcWIYswjgng6W8WMqwplHyHMcjnhzaVbIhs6rsVgTwGJawtbmD5q6aYnJCRq4zI68wzKOe7d6av8jwMGZS-dd8cdHj_XyShws1cWLhLJCaGW_vHKA0i-D1WTDeKSev31IQKCXBSRMB5URPkHu6dNDm1UyDiLGlJBx6iuKTizGeGhb4o2SQ1y1ISKyQRhKuNih0XgY40hn4x-6Scqu6_thPey-4F32BeTnbzNLGQ0wXE--dcIVRbAZMEB7wn7V4mRczG4gTG9ZnqarfZnHMPVUTZ0AxxZ2d_ifef1wQv9aRyKNRVLiBxvSqJtu7NnDMZT_U8mXCc_OIREi9wn-6esDbmxYYeBpNPR6Oq0fbsBbr--A_nVUz99MAAvkSnLhV3zdAHmMgjqNJNCmr2kPtGyle-GqdBx0LYQh4mtc5FdqLZUz9dz6gv_ERua952na3S1UnJr__8zVwofLEPGjiC5_g1nQuVCRE3QVSufMmw4gQL5kdUSuZGEm3RN__6DwAIS_X-E-aBv3utOMs92JWGandVEze8K7p4wQr3gSL5zlepjWHeYPZjiXCnGY=", + }, + { + "type": "text", + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.", + }, + { + "type": "tool_use", + "id": "call_H6dablHqZFScV63Y29mQXJvf", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], }, ) GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "openai/gpt-6-astra", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "openai/gpt-6-astra"}, expected_provider_endpoint="/responses", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, - expected_provider_request={ - "model": "gpt-6-astra", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "include": ["reasoning.encrypted_content"], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, "model": "gpt-6-astra"}, mock_provider_response={ - "id": "resp_023f789546179f0e006ad016c0984487d08dba1308aa780ef6", - "object": "response", - "created_at": 1792022208, - "status": "completed", - "access_programs": None, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1792022211, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6-astra", - "moderation": None, + "id": "resp_0964a41f58153ac5006ad0238f9bc087d0a9e79fda375bee3c", + "created_at": 1792025487, + "completed_at": 1792025491, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "messages client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect." + ), "output": [ { - "id": "msg_023f789546179f0e006ad016c2ef2c87d0b9d8f5ef59e691cd", + "id": "rs_0964a41f58153ac5006ad02391213887d0a1c21052bdd0c548", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COTU7GaEIVpMk00EB_rxvW069YSty7ozHW12516PLbLq0b935jCB0GevftK_27zbuIgpBhSTV-H3eiUTLx-CrGm41k8lA9CdDmFNKWBGef5fUtpfXIKNAPT3WRIkUe6ZrrMCLRa3zMhaitRJjxMf8RWRQGf0LqeomxdgNtJPU_YjiG2gapnmiUh8KSlqt30UqjcUVgjbw6isCgofeLdsvBDjy1GWfE81FTcapzObq-LzPlR6Lw838JvbjKJmBaT6B-g3UIy4ZpHCEOsLKRHdlSG97KHy84JwcRU4W1GB259oCQmWs7TRyA8BJao1vs5JrCmT0XdABCyrhUOERc7d8N0qBri6XJaVz3v1FmUObP1oz4I7Z47lqjX0a32_keE4AqjTP2evWOvjOnGsH5PVMsWknZWuvmKMYZefWuy0d7L5IUpPZ9pL4oZDgygSTH4nQ5t1ZWeYDIOKDf4oB0aTHAfgLO_IS-XgbRNMloPbeRm4acU9BI9KqGZShTVhBX34ZEc4ov7hpSmzTSUl1ukSsUmmxHqNLtKbz5CAZaYbBFM53KAyS6Bkexp6F1y8z6zKmy-LFENx-W8bZQIVaYmFAWwajro57XZE3qugHkWwCTOrP1HANsXIXXZ0wYhUdgvDqLxeRkOcaw7DAINL8K6zK71wt7Qc_j177tZjIr9uxXzEpQ3mu2p7Ji-ZBR6YRYx9sNrS55Xt5iMZYhW1X73I04pXIR5xsGO7hWZV1cGXCpWBXJ5vmMfpi4AXERmas0PAnDXEX4CRtdXflugJw7AvYDq5nY24xDYdCV9DuFF4VBzUXrWHG29OWbVzGwIegWrBEXiNIEoL4vJ05l_SqljbzKZtDg0kvY-6DnV7NwBkCYk6UgfRvJ5h8IKb7KPj7qcUhI3Zi7WXQf4yk2tH_MnM713aQ4ywDQorMJfzTcs11TABEPUwyuWt5_IHLB76c6bK97Uf8N7PkgseQ3BmPf9COF0K2JpyyUxBQOn8V5KjMK_2VTLGHQYsMA3AnX13t-_cde99USqDGqQa67C0EcbIbmHsAuctgYQP1rA_1GUB7A6oY2e1RUSoTNR3agm1FOnvAsVIjU9RW65gsI0uyz1VyIh6ZN3Ju7nqWldSDh00hIqT9bX6ctkSCZjb0Juhws_7yD7mIrykX51AfijJqCxOwI4clnITCrt2SLjuviYKxzfjCvj95107ui2fNVyWj7ksEFOX4gyA23ANATQgCWO0ITxVGnYAmZpiBSUNiJYsRUtCT6qQ1-NULhAn9UWuQJROqfJ4zACjY34kD9TtGhaLXJpKJjGw-YZ1iDUPGRG0NqPLBvI__8abyYLvA_une1qnEE49xseuR5-9uC51CpWr-Z5r40Q-Ner12_Em0p52GP52w3R14BHbuKqIDG0I1Xd47LjoNRLx7p25iivbyrcq-veIQJfKIPs4DGoAFkSyJe7z1kIuThQlusfYSaMffWXsOyWGS5jLsPSF8bRyUdfpR28wUqvbwkQFXs_5GRJLzAtBLSIJRW_SZ4mCt65B4kzkwzXJJBPvrmx0Vh2_rz-eufHW3S5S4VPsQg6nED3PB213LNzlMwwcm1KuEj0CpUXbffrKslKhEFfIFDdMchXP99i9UteCUipYdAWcXZCRYhAgMWQMdmEVVxtQRM34D-YVonMxnCQivNWYp99EhWGBfZZ-ffKV9c2WWboXUpVV_Oj8tqDIADyvRzJOd2-dGpDY2OdMzG31-ADUK-bXg68gHNSMRubpwPi7h_GBfYzPFCOFl3xaGRGE-6CH7AtzvHkZvUnrq-N5ArEv6hdhBA3qQZIh0rlyLXgNLbNz2KnP8lD7SOnGGz2sHVXoXHpW4Zu8V0coR6bxlAJ4Q5BTpcygg0q_BBWL2gr8uhxmP95XE0ANrRX8anoFoOmpS75iAsxBmkjYw8XyLWKUTj_4api4Xb1826PVTRW5x2rScJzmjVLM9AI3QiT05WSXE68NwOTO3GtYkNhDZYzBV6Aoya9f4lJwQKK76mfF8HRUen1AK4fD1X7cyI9NX9kisBrPS9YplUqF0PyFUsrL1aH6DXXmgrnsQ_V7QkMPEVwzd7NEFhHf1VbuJVzWMwzGK5OqROKxCjAY1mb34iK2r6LytJ6pX-oBIzKE7XfxcVTzzOutKdVnp_I3O9uI-BG-5Xj3degzcAdOZZbiPXNmh0kERrBgE620s6bAGiG5OGIfpS5mu4JbA1wL1nBfAC_pgdgh-62Kgo2GunkczL8Zrddxj_Xia_VBDyk6ptWYhETE4P-WWq-3wS2kUKEtuf_VPij-xi2imKOoF-NaA6rDUSoEKxqndBJsAZlAQOfX1OvkN8q44Am4RiZkzEfQEbHlde7ziC1WCVU9zsjWXe8V4EJ78CxWENhtbmF3BhUHTYaywoItp0PhYf1l7FVI7FWxXQCOX5h5MvOIzr4nf-8dMyMRR7q35aHyosc6RqOGY6OofOBcDcqFHpHAcW602H-vS2_fj7fOqCqBDmqcmEriPIBFbczBbFFZQ==", + "summary": [], + }, + { + "id": "msg_0964a41f58153ac5006ad02392bf8487d0ba759f0a48ebb3c9", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.\n", + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_0964a41f58153ac5006ad0239340f887d0a46fc41789aa1a55", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_eqYF3Vli8ZUYaCLw3MAUvxKq", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 152, + "output_tokens_details": {"reasoning_tokens": 87}, + "total_tokens": 461, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ - "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDphODRkZDk3MTcwNzk0ZDlhYzQ5ODhlZWMzOTJmN2Q5YjQwN2MyNDY1YmJhZTA4MGE1ZGYzMThlOTViOTA2NzlkO3Jlc3BvbnNlX2lkOnJlc3BfMDIzZjc4OTU0NjE3OWYwZTAwNmFkMDE2YzA5ODQ0ODdkMDhkYmExMzA4YWE3ODBlZjY=", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDphODRkZDk3MTcwNzk0ZDlhYzQ5ODhlZWMzOTJmN2Q5YjQwN2MyNDY1YmJhZTA4MGE1ZGYzMThlOTViOTA2NzlkO3Jlc3BvbnNlX2lkOnJlc3BfMDk2NGE0MWY1ODE1M2FjNTAwNmFkMDIzOGY5YmMwODdkMGE5ZTc5ZmRhMzc1YmVlM2M=", "model": "openai/gpt-6-astra", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 152}, + "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:gAAAAABq0COTU7GaEIVpMk00EB_rxvW069YSty7ozHW12516PLbLq0b935jCB0GevftK_27zbuIgpBhSTV-H3eiUTLx-CrGm41k8lA9CdDmFNKWBGef5fUtpfXIKNAPT3WRIkUe6ZrrMCLRa3zMhaitRJjxMf8RWRQGf0LqeomxdgNtJPU_YjiG2gapnmiUh8KSlqt30UqjcUVgjbw6isCgofeLdsvBDjy1GWfE81FTcapzObq-LzPlR6Lw838JvbjKJmBaT6B-g3UIy4ZpHCEOsLKRHdlSG97KHy84JwcRU4W1GB259oCQmWs7TRyA8BJao1vs5JrCmT0XdABCyrhUOERc7d8N0qBri6XJaVz3v1FmUObP1oz4I7Z47lqjX0a32_keE4AqjTP2evWOvjOnGsH5PVMsWknZWuvmKMYZefWuy0d7L5IUpPZ9pL4oZDgygSTH4nQ5t1ZWeYDIOKDf4oB0aTHAfgLO_IS-XgbRNMloPbeRm4acU9BI9KqGZShTVhBX34ZEc4ov7hpSmzTSUl1ukSsUmmxHqNLtKbz5CAZaYbBFM53KAyS6Bkexp6F1y8z6zKmy-LFENx-W8bZQIVaYmFAWwajro57XZE3qugHkWwCTOrP1HANsXIXXZ0wYhUdgvDqLxeRkOcaw7DAINL8K6zK71wt7Qc_j177tZjIr9uxXzEpQ3mu2p7Ji-ZBR6YRYx9sNrS55Xt5iMZYhW1X73I04pXIR5xsGO7hWZV1cGXCpWBXJ5vmMfpi4AXERmas0PAnDXEX4CRtdXflugJw7AvYDq5nY24xDYdCV9DuFF4VBzUXrWHG29OWbVzGwIegWrBEXiNIEoL4vJ05l_SqljbzKZtDg0kvY-6DnV7NwBkCYk6UgfRvJ5h8IKb7KPj7qcUhI3Zi7WXQf4yk2tH_MnM713aQ4ywDQorMJfzTcs11TABEPUwyuWt5_IHLB76c6bK97Uf8N7PkgseQ3BmPf9COF0K2JpyyUxBQOn8V5KjMK_2VTLGHQYsMA3AnX13t-_cde99USqDGqQa67C0EcbIbmHsAuctgYQP1rA_1GUB7A6oY2e1RUSoTNR3agm1FOnvAsVIjU9RW65gsI0uyz1VyIh6ZN3Ju7nqWldSDh00hIqT9bX6ctkSCZjb0Juhws_7yD7mIrykX51AfijJqCxOwI4clnITCrt2SLjuviYKxzfjCvj95107ui2fNVyWj7ksEFOX4gyA23ANATQgCWO0ITxVGnYAmZpiBSUNiJYsRUtCT6qQ1-NULhAn9UWuQJROqfJ4zACjY34kD9TtGhaLXJpKJjGw-YZ1iDUPGRG0NqPLBvI__8abyYLvA_une1qnEE49xseuR5-9uC51CpWr-Z5r40Q-Ner12_Em0p52GP52w3R14BHbuKqIDG0I1Xd47LjoNRLx7p25iivbyrcq-veIQJfKIPs4DGoAFkSyJe7z1kIuThQlusfYSaMffWXsOyWGS5jLsPSF8bRyUdfpR28wUqvbwkQFXs_5GRJLzAtBLSIJRW_SZ4mCt65B4kzkwzXJJBPvrmx0Vh2_rz-eufHW3S5S4VPsQg6nED3PB213LNzlMwwcm1KuEj0CpUXbffrKslKhEFfIFDdMchXP99i9UteCUipYdAWcXZCRYhAgMWQMdmEVVxtQRM34D-YVonMxnCQivNWYp99EhWGBfZZ-ffKV9c2WWboXUpVV_Oj8tqDIADyvRzJOd2-dGpDY2OdMzG31-ADUK-bXg68gHNSMRubpwPi7h_GBfYzPFCOFl3xaGRGE-6CH7AtzvHkZvUnrq-N5ArEv6hdhBA3qQZIh0rlyLXgNLbNz2KnP8lD7SOnGGz2sHVXoXHpW4Zu8V0coR6bxlAJ4Q5BTpcygg0q_BBWL2gr8uhxmP95XE0ANrRX8anoFoOmpS75iAsxBmkjYw8XyLWKUTj_4api4Xb1826PVTRW5x2rScJzmjVLM9AI3QiT05WSXE68NwOTO3GtYkNhDZYzBV6Aoya9f4lJwQKK76mfF8HRUen1AK4fD1X7cyI9NX9kisBrPS9YplUqF0PyFUsrL1aH6DXXmgrnsQ_V7QkMPEVwzd7NEFhHf1VbuJVzWMwzGK5OqROKxCjAY1mb34iK2r6LytJ6pX-oBIzKE7XfxcVTzzOutKdVnp_I3O9uI-BG-5Xj3degzcAdOZZbiPXNmh0kERrBgE620s6bAGiG5OGIfpS5mu4JbA1wL1nBfAC_pgdgh-62Kgo2GunkczL8Zrddxj_Xia_VBDyk6ptWYhETE4P-WWq-3wS2kUKEtuf_VPij-xi2imKOoF-NaA6rDUSoEKxqndBJsAZlAQOfX1OvkN8q44Am4RiZkzEfQEbHlde7ziC1WCVU9zsjWXe8V4EJ78CxWENhtbmF3BhUHTYaywoItp0PhYf1l7FVI7FWxXQCOX5h5MvOIzr4nf-8dMyMRR7q35aHyosc6RqOGY6OofOBcDcqFHpHAcW602H-vS2_fj7fOqCqBDmqcmEriPIBFbczBbFFZQ==", + }, + { + "type": "text", + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.\n", + }, + { + "type": "tool_use", + "id": "call_eqYF3Vli8ZUYaCLw3MAUvxKq", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], + }, +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "openai/gpt-5.6-terra"}, + expected_provider_endpoint="/responses", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, "model": "gpt-5.6-terra"}, + mock_provider_response={ + **OPENAI_RESPONSES_RESPONSE, + "model": "gpt-5.6-terra", + "id": "resp_08526ccab9aa653e006ad023884a1c87d09a23f6d5c61d65d3", + "created_at": 1792025480, + "access_programs": {"cyber": "daybreak_blue"}, + "completed_at": 1792025481, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "messages client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect." + ), + "output": [ + { + "id": "rs_08526ccab9aa653e006ad02388d60087d0909f13870c67fc8c", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COJ5vm7lH4CvuQJznQ9B0y2iQwRylfHyiFtKnR3At3jt1oCNRl_9-mCg3S5oBCgnMLM9u4EviSSArrlNlXBn1QqjilF0jg47ecCqzcLjNkSXQfvCFKM3N9QUb33bNkruIA9VTZFJb2Dh4CWd_8uP4W2lHVfLiTcp5WGy301lO_5a8hbuO2sv08S4I0Ug67I-EFi_BMZyauCNqY8Zmbf7CodUc6cDluIeFClKOnXPaM-I4SunKVAvTZz_BsvmO-hhV2ajhBia68VqMhlioEyJQR0g5ouQrY_mhhMs99sOn4-V8aIrlkDfIGz4SEwXGRd55MSGRa095JCZig1Jzm_CdublE9Dkc0EsbJB6xjrJZfAyJZVGB95xOOsWHafy0MW878IqJNivjj6XULFSHOuVwm3WqA4_L0GKv5o0eor3OYggauA4AgKx0FiTaUAlCm5tNy2qSGMzCcKWp85CrItPYZUhWgkn4lcwGhGpJ20kmOHQbOGxnMtSpaTJL6PuMC0oyXWJli9qjaaIaB5cGhvoOtkBzifPh8pSZVsRNAnO7q7S72Lk4MtpRqCVIAVeB6kXTmkqMLhLghRowcd8xemlswAOCLqnR2usZncZIbY3kGqprxWA7u9w_acwISRcQZE6rmnefcK4I7NpA2zEuVcEVJndBWZbApBZE_OZFg7doNuw9cqhk-4y3VePW6fjSA8e68gX0VslTHODd2W6o16LQuGA-Zp84T0W0NuYI0Y2zY6AEY63DW2fDbKlbvRc5Shm5sPIugD0bWsR2SRkEMgvFnYalZFlwSojKIsxw1ET55tXqjiP4TLg1-Q4vnSkIR4dxP6lubbeN-SLCY2Nq6MVW5rqnh6o_AkFZHemKqdjFNzy5lR3JYs_4bhYetICCYSqLQVp8Cy1wmUkNeleyPjayQJFAF7lhY4mN9VUJxut-H7aizH9W0QGfoIukNLJOc3gZ7svAa3jElXtTef8Qv-rmQFA_y-AMRfHZeDLzhtxQnaDmKDeiqimbREwqG1BGvQeEKWCGPba7RGNuks9yYhNvi5NyVsEYXJe5Omy-pTZ47vrRUinlR4ecl75NPz1bL-lLtYhIMtvLtuJvkJ_ieD6g7oP3qEcsS34ARmBq6ZpEkelgL9ifh8DGX9gerq-JbsmnyI-c8aG9Go--hJWXs2dKJSfBq1ABLhhW3rK3Ms5163BkdtZp67F0LPISjU8i4Kh_wSxZbqXkdJy-4oYgAqG_YGQMz5tyM3GFMj-ozDuznqivj3GcD7xRTTA2CfDTbFxT1mrTpLv_NjKsfjMOSH5KF5RLwP8fatr76F0YUKW75korD9RG0tJpbSHBq67r4u0jwyNRJFZMPsmvOohr4GaTOW3-kBfUZqVU7x2X6cjmfw50jY4EOQb1r4sJBbVkgqTcw36zPSzR46dkJHUtEZMqNgb0u8AUze_Z1Or3x0DInw6xMSAs2jzYNEiIYzJ0EJ62_ciEcWT5aYUGSZGhNy2i9dIEJiOTYfzL-b7dNzs-6MoWMHzMaPPPcp4e8BLCgDJd6ryDuuHRd7NZmEnt2jdUlKr4iqEstENijjJjdl1xolCtUMPYQahG9iYPMa2tKqXAZYTyf6_cEdv6_7kUdFd6iUuU7kOplokkOi9A-GbA0Lhhct5HP3QYxMSymUoMv_NO6xjgAImYwesVohnAaN1hNmraWuavZKj4w95byYav83k0xFB_mz3xKYjLsCtTw3rF90UsWk4KgXnoZRZv0FcesyyNAXfzizkgsI0Ndt4i8EIHhP6upFi473ZMqbWo5_RQtAiKsHBD1L5NG51Lai2caL5bnvOrHtfW_aZcLGNdFQbziTpTFwafefusBSGR0hLHc0LzKpMg-XDCwzU8AnK3yngTTKnOb9j0_avEdz5xv9xnLlp5rnHtMrYRna-5HPQ2xKIXmGzbMx2h8OQYdWTqmVayubLoc-KpW0qKYxYJRRIgo=", + "summary": [], + }, + { + "id": "msg_08526ccab9aa653e006ad023892c7087d0870b4b072127659f", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Next I need `shop/validation.py` to inspect the validation branches for " + "zero, negative, and non-integer quantities.", + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_08526ccab9aa653e006ad02389692887d0ab14a9eaef763aee", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_iom4KPAiRrGmsYIsEHwM2vZo", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 107, + "output_tokens_details": {"reasoning_tokens": 42}, + "total_tokens": 416, + }, + }, + expected_litellm_response={ + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo0Mjg5MWEzMzRjNTlhMjhlMjQ2MjQ0YTQ4YTJjOGVlMTM5NGYyZGNhZWMxYmVjZDhmZDk2N2NmNjYxNWNmZDM5O3Jlc3BvbnNlX2lkOnJlc3BfMDg1MjZjY2FiOWFhNjUzZTAwNmFkMDIzODg0YTFjODdkMDlhMjNmNmQ1YzYxZDY1ZDM=", + "model": "openai/gpt-5.6-terra", + "usage": {"input_tokens": 309, "output_tokens": 107}, + "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:gAAAAABq0COJ5vm7lH4CvuQJznQ9B0y2iQwRylfHyiFtKnR3At3jt1oCNRl_9-mCg3S5oBCgnMLM9u4EviSSArrlNlXBn1QqjilF0jg47ecCqzcLjNkSXQfvCFKM3N9QUb33bNkruIA9VTZFJb2Dh4CWd_8uP4W2lHVfLiTcp5WGy301lO_5a8hbuO2sv08S4I0Ug67I-EFi_BMZyauCNqY8Zmbf7CodUc6cDluIeFClKOnXPaM-I4SunKVAvTZz_BsvmO-hhV2ajhBia68VqMhlioEyJQR0g5ouQrY_mhhMs99sOn4-V8aIrlkDfIGz4SEwXGRd55MSGRa095JCZig1Jzm_CdublE9Dkc0EsbJB6xjrJZfAyJZVGB95xOOsWHafy0MW878IqJNivjj6XULFSHOuVwm3WqA4_L0GKv5o0eor3OYggauA4AgKx0FiTaUAlCm5tNy2qSGMzCcKWp85CrItPYZUhWgkn4lcwGhGpJ20kmOHQbOGxnMtSpaTJL6PuMC0oyXWJli9qjaaIaB5cGhvoOtkBzifPh8pSZVsRNAnO7q7S72Lk4MtpRqCVIAVeB6kXTmkqMLhLghRowcd8xemlswAOCLqnR2usZncZIbY3kGqprxWA7u9w_acwISRcQZE6rmnefcK4I7NpA2zEuVcEVJndBWZbApBZE_OZFg7doNuw9cqhk-4y3VePW6fjSA8e68gX0VslTHODd2W6o16LQuGA-Zp84T0W0NuYI0Y2zY6AEY63DW2fDbKlbvRc5Shm5sPIugD0bWsR2SRkEMgvFnYalZFlwSojKIsxw1ET55tXqjiP4TLg1-Q4vnSkIR4dxP6lubbeN-SLCY2Nq6MVW5rqnh6o_AkFZHemKqdjFNzy5lR3JYs_4bhYetICCYSqLQVp8Cy1wmUkNeleyPjayQJFAF7lhY4mN9VUJxut-H7aizH9W0QGfoIukNLJOc3gZ7svAa3jElXtTef8Qv-rmQFA_y-AMRfHZeDLzhtxQnaDmKDeiqimbREwqG1BGvQeEKWCGPba7RGNuks9yYhNvi5NyVsEYXJe5Omy-pTZ47vrRUinlR4ecl75NPz1bL-lLtYhIMtvLtuJvkJ_ieD6g7oP3qEcsS34ARmBq6ZpEkelgL9ifh8DGX9gerq-JbsmnyI-c8aG9Go--hJWXs2dKJSfBq1ABLhhW3rK3Ms5163BkdtZp67F0LPISjU8i4Kh_wSxZbqXkdJy-4oYgAqG_YGQMz5tyM3GFMj-ozDuznqivj3GcD7xRTTA2CfDTbFxT1mrTpLv_NjKsfjMOSH5KF5RLwP8fatr76F0YUKW75korD9RG0tJpbSHBq67r4u0jwyNRJFZMPsmvOohr4GaTOW3-kBfUZqVU7x2X6cjmfw50jY4EOQb1r4sJBbVkgqTcw36zPSzR46dkJHUtEZMqNgb0u8AUze_Z1Or3x0DInw6xMSAs2jzYNEiIYzJ0EJ62_ciEcWT5aYUGSZGhNy2i9dIEJiOTYfzL-b7dNzs-6MoWMHzMaPPPcp4e8BLCgDJd6ryDuuHRd7NZmEnt2jdUlKr4iqEstENijjJjdl1xolCtUMPYQahG9iYPMa2tKqXAZYTyf6_cEdv6_7kUdFd6iUuU7kOplokkOi9A-GbA0Lhhct5HP3QYxMSymUoMv_NO6xjgAImYwesVohnAaN1hNmraWuavZKj4w95byYav83k0xFB_mz3xKYjLsCtTw3rF90UsWk4KgXnoZRZv0FcesyyNAXfzizkgsI0Ndt4i8EIHhP6upFi473ZMqbWo5_RQtAiKsHBD1L5NG51Lai2caL5bnvOrHtfW_aZcLGNdFQbziTpTFwafefusBSGR0hLHc0LzKpMg-XDCwzU8AnK3yngTTKnOb9j0_avEdz5xv9xnLlp5rnHtMrYRna-5HPQ2xKIXmGzbMx2h8OQYdWTqmVayubLoc-KpW0qKYxYJRRIgo=", + }, + { + "type": "text", + "text": "Next I need `shop/validation.py` to inspect the validation branches for " + "zero, negative, and non-integer quantities.", + }, + { + "type": "tool_use", + "id": "call_iom4KPAiRrGmsYIsEHwM2vZo", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], }, ) diff --git a/tests/integration/translation/messages/bases/openai_responses.py b/tests/integration/translation/messages/bases/openai_responses.py index ff42bb5b8aa..f5a26684ea6 100644 --- a/tests/integration/translation/messages/bases/openai_responses.py +++ b/tests/integration/translation/messages/bases/openai_responses.py @@ -1,6 +1,8 @@ from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_BRIDGE_RESPONSE, MESSAGES_REQUEST +from integration.translation.templates.openai import OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, OPENAI_RESPONSES_RESPONSE GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -100,94 +102,122 @@ GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "openai/responses/gpt-6.1-sol", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "openai/responses/gpt-6.1-sol"}, expected_provider_endpoint="/responses", - expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"}, - expected_provider_request={ - "model": "gpt-6.1-sol", - "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], - "include": ["reasoning.encrypted_content"], - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, "model": "gpt-6.1-sol"}, mock_provider_response={ - "id": "resp_03446eafdb61cd31006ac3f046b20c87d0af12e5a0e55548fd", - "object": "response", - "created_at": 1791225926, - "status": "completed", - "access_programs": None, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791225928, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6.1-sol", - "moderation": None, + "id": "resp_0af67cb78bead2c6006ad023a7273487d0b1768b77f6956b73", + "created_at": 1792025511, + "completed_at": 1792025514, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "messages client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect." + ), "output": [ { - "id": "msg_03446eafdb61cd31006ac3f048804087d0a22e0e94d0a6d50e", + "id": "rs_0af67cb78bead2c6006ad023a85d5c87d0810de0f56bf80208", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COqmCc2HPj-8NSt1bzfstH4-YXiTtNvZLyVMciprSqFmNbPXwcPXWrps8lsryqoAY5MuRQ_pO9yNIsCVBRlHXitSdObtJRB5Ld7TGQnbUekIsl2TNIFPsnnc8PYSdVrJlcG3FZmykyeSVHblM5F9SaPzHFzOfvCdYQtN52yt2lPhjYailjRidMwkl4KDGQ1BlcR3U29KKxo0020AqMMvsAht5kx5aAFaKfaIum5jctQrjgL8Y99h6sV3gc3LwI0W6ZR2IIqs6e6ADrVIGdVmSpn2GZTHmz6vUFaROU2-UTbVL7QrCy0BUg5CEqjnZ563lqsEYuWOxTVK0RXCPonWwmhfSllKffjLxtGuaq4dtl6v3lGIt-BW3-gJ1WqhtfTDV5VWTAliJhx09kgGhXz7VuxQRQMBSy--M9nBhdNJ1KfkbC0YDUcq5LT_cKh-KZU_-jNWCSQa7hvNJP-aCQ85oL-av-9fT5Wb5ovLUzELiBDooQLH8SalXQgFo-GX_wvSF-IH7byXwCka0F9fyIW6FjmRkirX8h_WF7pLDJeZDO5_8BWbxGOxDI6hEURHW9HLGG-e8S_lp1cVc7i37r-3KNqOwVlnlxEn2hj9TmDvTxRk4yv71AoSuwF8j3Wsh-im1HgmsNf9sdKZIZUTJTdvGCNuf54xXedUEdfoInFspUuiLXj_5KwWpckwJD0y1Fs8oEfTq2Y0Bn3LdBvwrAGgijcwnDsj6nRa2L9oMLrbHROG3xWIuTC6JyKIlVo8o7cGsM6XdxV6P9f3Wc-UgScNzc-fleF3oWzVq2gENtufLiwn9y3z1k_Hb62pbgIV6dmhLepqVH7WAyHajYt9GyxHB_FZbni-_rujbci3-3Jcj451W8PFg0TjhlyUzETBQwv6y6-zORZcbD7TqKe6XnUUGxHEMbPKBCmiIxOIUuOHpNOEjmdQpgOuPR16k_j0YjoA1gJJovIqwB1g6w44nebjzzsky0uRWmIhqeNaIFPPtIUDr05z1ocxgFeHJxum5YAh4d46vUY18dpffRyghwrd1rvIxk2EWW2rmruR4nEDTT405t3RX4kjnhGHuyShSLM_jinQm5LcpLDXUWgRnilpB1RVB8eZUSNYePAhgDRbacr8IC4AvQ7X8xUtcglbDJmMYQZQRFrkGfVoMUcyAkEJamLjccYDFrO5rxUFfyUHmGVxFUY85NW9XCwAFOx7h3606gjN9KVKQPMAg-dUnXbfhc4PCY3iRaS0djErOEugRN6f7iniy8td2YRI2WhPD5O1-qsvTDfiIhymTFq2L0v4f3UEeIa2wnfq4EdnAgla4D14w7T1D1Wg4SiP-dnF0_vKy8md5vlQjpbPGbYJkzB9DBNwJ8qR-7s1jCtZj2fViHuIGwTReI3OXUoeao4FHC8pndoAZbEpbTmax7BEYw7JqlUhIXWF2x8NzeYr-sjBbo1kbzKbtnidxOwrngjU428TzTPEjr9WA9VpmxXNoSOPL7I_qsN8ulnGegsnxQOEIarRK9CE7C8AwtMRMi8lClAFaMwMAoD8jWMNCSEa6mtyv08AikNCt3ij6cW2JszTRO3Qu4lRTsSnta41Qyk16Zc4xEPOs58PhY90t_atHYUWRpydUerE2nvRnPF9eTbQvSe8-iZ0BqkXqdlnVAAIzwOIfVZRua0p8eU3iPlzKteg9F02Jb5HkOsvi2Y4M-ZS5i0Tg4iOeuKHkeiFdZs6EMytIwz6lPGKLhYii6HW0Oe0KU_WQrIVNdXMcs5EWZGqMIdEhf09ZpO3ONcWIYswjgng6W8WMqwplHyHMcjnhzaVbIhs6rsVgTwGJawtbmD5q6aYnJCRq4zI68wzKOe7d6av8jwMGZS-dd8cdHj_XyShws1cWLhLJCaGW_vHKA0i-D1WTDeKSev31IQKCXBSRMB5URPkHu6dNDm1UyDiLGlJBx6iuKTizGeGhb4o2SQ1y1ISKyQRhKuNih0XgY40hn4x-6Scqu6_thPey-4F32BeTnbzNLGQ0wXE--dcIVRbAZMEB7wn7V4mRczG4gTG9ZnqarfZnHMPVUTZ0AxxZ2d_ifef1wQv9aRyKNRVLiBxvSqJtu7NnDMZT_U8mXCc_OIREi9wn-6esDbmxYYeBpNPR6Oq0fbsBbr--A_nVUz99MAAvkSnLhV3zdAHmMgjqNJNCmr2kPtGyle-GqdBx0LYQh4mtc5FdqLZUz9dz6gv_ERua952na3S1UnJr__8zVwofLEPGjiC5_g1nQuVCRE3QVSufMmw4gQL5kdUSuZGEm3RN__6DwAIS_X-E-aBv3utOMs92JWGandVEze8K7p4wQr3gSL5zlepjWHeYPZjiXCnGY=", + "summary": [], + }, + { + "id": "msg_0af67cb78bead2c6006ad023a9b1e487d09ad5bb0d5cc96bde", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.", + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_0af67cb78bead2c6006ad023aa22c487d0b76512608a221ae3", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_H6dablHqZFScV63Y29mQXJvf", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 140, + "output_tokens_details": {"reasoning_tokens": 75}, + "total_tokens": 449, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ - "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo3ZGE3NDY3Njc3N2IyZjVkYmQ3MDI1MTI1ZjEzMDdlYWQ0MjM3YTgzN2QxZTJmZDhkY2FkNjhlODM4ODU1Zjg2O3Jlc3BvbnNlX2lkOnJlc3BfMDM0NDZlYWZkYjYxY2QzMTAwNmFjM2YwNDZiMjBjODdkMGFmMTJlNWEwZTU1NTQ4ZmQ=", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo3ZGE3NDY3Njc3N2IyZjVkYmQ3MDI1MTI1ZjEzMDdlYWQ0MjM3YTgzN2QxZTJmZDhkY2FkNjhlODM4ODU1Zjg2O3Jlc3BvbnNlX2lkOnJlc3BfMGFmNjdjYjc4YmVhZDJjNjAwNmFkMDIzYTcyNzM0ODdkMGIxNzY4Yjc3ZjY5NTZiNzM=", "model": "openai/responses/gpt-6.1-sol", - "stop_sequence": None, - "usage": {"input_tokens": 19, "output_tokens": 6}, - "content": [{"type": "text", "text": "Hello!"}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 309, "output_tokens": 140}, + "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:gAAAAABq0COqmCc2HPj-8NSt1bzfstH4-YXiTtNvZLyVMciprSqFmNbPXwcPXWrps8lsryqoAY5MuRQ_pO9yNIsCVBRlHXitSdObtJRB5Ld7TGQnbUekIsl2TNIFPsnnc8PYSdVrJlcG3FZmykyeSVHblM5F9SaPzHFzOfvCdYQtN52yt2lPhjYailjRidMwkl4KDGQ1BlcR3U29KKxo0020AqMMvsAht5kx5aAFaKfaIum5jctQrjgL8Y99h6sV3gc3LwI0W6ZR2IIqs6e6ADrVIGdVmSpn2GZTHmz6vUFaROU2-UTbVL7QrCy0BUg5CEqjnZ563lqsEYuWOxTVK0RXCPonWwmhfSllKffjLxtGuaq4dtl6v3lGIt-BW3-gJ1WqhtfTDV5VWTAliJhx09kgGhXz7VuxQRQMBSy--M9nBhdNJ1KfkbC0YDUcq5LT_cKh-KZU_-jNWCSQa7hvNJP-aCQ85oL-av-9fT5Wb5ovLUzELiBDooQLH8SalXQgFo-GX_wvSF-IH7byXwCka0F9fyIW6FjmRkirX8h_WF7pLDJeZDO5_8BWbxGOxDI6hEURHW9HLGG-e8S_lp1cVc7i37r-3KNqOwVlnlxEn2hj9TmDvTxRk4yv71AoSuwF8j3Wsh-im1HgmsNf9sdKZIZUTJTdvGCNuf54xXedUEdfoInFspUuiLXj_5KwWpckwJD0y1Fs8oEfTq2Y0Bn3LdBvwrAGgijcwnDsj6nRa2L9oMLrbHROG3xWIuTC6JyKIlVo8o7cGsM6XdxV6P9f3Wc-UgScNzc-fleF3oWzVq2gENtufLiwn9y3z1k_Hb62pbgIV6dmhLepqVH7WAyHajYt9GyxHB_FZbni-_rujbci3-3Jcj451W8PFg0TjhlyUzETBQwv6y6-zORZcbD7TqKe6XnUUGxHEMbPKBCmiIxOIUuOHpNOEjmdQpgOuPR16k_j0YjoA1gJJovIqwB1g6w44nebjzzsky0uRWmIhqeNaIFPPtIUDr05z1ocxgFeHJxum5YAh4d46vUY18dpffRyghwrd1rvIxk2EWW2rmruR4nEDTT405t3RX4kjnhGHuyShSLM_jinQm5LcpLDXUWgRnilpB1RVB8eZUSNYePAhgDRbacr8IC4AvQ7X8xUtcglbDJmMYQZQRFrkGfVoMUcyAkEJamLjccYDFrO5rxUFfyUHmGVxFUY85NW9XCwAFOx7h3606gjN9KVKQPMAg-dUnXbfhc4PCY3iRaS0djErOEugRN6f7iniy8td2YRI2WhPD5O1-qsvTDfiIhymTFq2L0v4f3UEeIa2wnfq4EdnAgla4D14w7T1D1Wg4SiP-dnF0_vKy8md5vlQjpbPGbYJkzB9DBNwJ8qR-7s1jCtZj2fViHuIGwTReI3OXUoeao4FHC8pndoAZbEpbTmax7BEYw7JqlUhIXWF2x8NzeYr-sjBbo1kbzKbtnidxOwrngjU428TzTPEjr9WA9VpmxXNoSOPL7I_qsN8ulnGegsnxQOEIarRK9CE7C8AwtMRMi8lClAFaMwMAoD8jWMNCSEa6mtyv08AikNCt3ij6cW2JszTRO3Qu4lRTsSnta41Qyk16Zc4xEPOs58PhY90t_atHYUWRpydUerE2nvRnPF9eTbQvSe8-iZ0BqkXqdlnVAAIzwOIfVZRua0p8eU3iPlzKteg9F02Jb5HkOsvi2Y4M-ZS5i0Tg4iOeuKHkeiFdZs6EMytIwz6lPGKLhYii6HW0Oe0KU_WQrIVNdXMcs5EWZGqMIdEhf09ZpO3ONcWIYswjgng6W8WMqwplHyHMcjnhzaVbIhs6rsVgTwGJawtbmD5q6aYnJCRq4zI68wzKOe7d6av8jwMGZS-dd8cdHj_XyShws1cWLhLJCaGW_vHKA0i-D1WTDeKSev31IQKCXBSRMB5URPkHu6dNDm1UyDiLGlJBx6iuKTizGeGhb4o2SQ1y1ISKyQRhKuNih0XgY40hn4x-6Scqu6_thPey-4F32BeTnbzNLGQ0wXE--dcIVRbAZMEB7wn7V4mRczG4gTG9ZnqarfZnHMPVUTZ0AxxZ2d_ifef1wQv9aRyKNRVLiBxvSqJtu7NnDMZT_U8mXCc_OIREi9wn-6esDbmxYYeBpNPR6Oq0fbsBbr--A_nVUz99MAAvkSnLhV3zdAHmMgjqNJNCmr2kPtGyle-GqdBx0LYQh4mtc5FdqLZUz9dz6gv_ERua952na3S1UnJr__8zVwofLEPGjiC5_g1nQuVCRE3QVSufMmw4gQL5kdUSuZGEm3RN__6DwAIS_X-E-aBv3utOMs92JWGandVEze8K7p4wQr3gSL5zlepjWHeYPZjiXCnGY=", + }, + { + "type": "text", + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.", + }, + { + "type": "tool_use", + "id": "call_H6dablHqZFScV63Y29mQXJvf", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 200}, + }, + ], }, ) diff --git a/tests/integration/translation/messages/bases/vertex_ai.py b/tests/integration/translation/messages/bases/vertex_ai.py index f52d096b3fb..6f506b39d93 100644 --- a/tests/integration/translation/messages/bases/vertex_ai.py +++ b/tests/integration/translation/messages/bases/vertex_ai.py @@ -1,13 +1,16 @@ from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_BRIDGE_RESPONSE, MESSAGES_REQUEST +from integration.translation.templates.vertex_ai import ( + VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, + VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, +) GEMINI_3_5_FLASH_THOUGHT_SIGNATURE: Final = "AY89a1+kriNs9YsPegJQxgjTWsMlxsqPU/UkHDyz4EK+3FRrE52ygqPIHTvkblgKKiO5Fn8R9D3iu//i+YnuuZ+e7W0yxUoFScas89b5rn3vZHsihnUA7VTe3b7beth1IftteoTGZ3F8OSLrSIP8HKn4xTIx35eeNEt5oiD3K5rTQrdklqO277HvTDhu/LuZhZ0pzaWciGQEc61epIBBpC2ygu8V4L8TekUVZnz6qzSzxipdiQStQ8dZou43334hMnxOR6JjJCGxqt/1K+C0vXX6YP6f+Bph5VhPSnvy2Z1uApi3TUxAcsDDyVewIr+ha2G6DPhN1Oi5uZZNhmbvH01Uq5OOquF1uEICq2m/35LhJIbDNM9N7oI1Z6L94P9xdugeFXM2S7wdJfc8sdg09vEzNeA++GK25zRuB4hU7XyaG4XnWjmu2kswi4sf+EyrO/K0P2tb/ZCJu7kJTtWTiD1wgSIl1eqS8ORKKsXDN2hIbFrtsNR50WIvVzd6" -GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = "AY89a1+gmp4bVJhwGOCtbUalmyfhwvgE4EsZOwS4bNR02vurTt4AZC72ypTNLJDRwK8I518tGYBiDnFqD6jTxkjWTQTPGwUZ5IOYr1IskTb3pPKfvZWZ/XunV7kfb9VmEU0TBlgwjVqxXziOxvjediJyb2gbFnCVjGTSVsq0ipmpFznF22lV1I6N9GI7RKBFgYXaSzGU1+fGbLLBigaUD0qbgquAnFv6R3JGpaLcW7EUrhKDinHoFukmUwBLx07S7SfuWBT8hMLMs32vEVzwpns9JmbVtuQG24zs5qQx9sJqtO9E07k6SFL3f1tvaKZsQ6Yjb08Xcuvnh9Pcz4ChCdRJ4vRkmtRW3AgOli8/43rrrAMmvIOqEdAOOvZW3W9D8lnOC+Nfq7S0FTNWPYkIJhrtskWoQkUwDrY6TOzMjIA6Jblt2f9kKU3VJzq7O1hPjbHsJKmLBYP+De8rKx7Ufq9Shs5qLLu4aBBgHleiyFiHGQSD18RDjnyv0DLRyrp7LOkjIYWBYoIA2+BSb0ySMyeLhk8B2U8XZ6AXiw3aHoNUG/wwuE0EhXBhOUBM14FmchGfptXUr+Dn9kZvPAy8mfbq3Pk=" - -GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = "AY89a198sdR5Cy3qgtvOq/o5xsmH1g3WymzyZyB/bv3XeyqNZAgXbCocCAqUS913U8FvVUodNrm3y2B5P/mDsn3A3pINhbEiwJcWA/o8bNJq+Z/1O2UYdViO0FMhOOS1aRaCbza+6aSBLPVQ81QnyxrwH6rdDCuQlwyZ+5/Evc/EOXLVxdUF9PYwTwNJcZPREl4UMcarKvtKsNofkSfa09EWQNVVMDkasNl3YHtQJqYLu5rL7vSUrbMtLLA0sMEMh2/4zMj4gIvtjoOwiI4CBdLLH0Xp8SGGco8UljxikaIp6YvQW4cHsfkaqTBQmLgREudbnKKTZ0MvMR52G/++7r8Iv6BL29+73igMMyHoQad40OM8RY3E6akpSE5uPXH/CgYTmSLXQONtRMXzrXm996JYLaHLOo2dP1RtaJANcjNoDB4eN5vj4GOpTUYqZqRRaB5jMX2od8QcAnaXk2AYs/2rNMaLV4zgraz45gbA0It465nN5hb4JbRH2toJ9K/TvsZY/YyoLmV9RjyAmIhbq6bG0fGatsXgmDaXj/vEsBmqfRyBN31k" - CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -244,126 +247,6 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/claude-sonnet-5-5@default", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-sonnet-5-5@default:rawPredict", - expected_provider_headers={"authorization": "Bearer scripted-token", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": "Say hello."}], - "max_tokens": 64, - "stream": False, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "vertex-2023-10-16", - }, - mock_provider_response={ - "model": "claude-sonnet-5-5", - "id": "msg_vrtx_011CfjuysbBy7Cc5quJ5tS6c", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - }, - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "model": "vertex_ai/claude-sonnet-5-5@default", - "id": "msg_vrtx_011CfjuysbBy7Cc5quJ5tS6c", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - }, - }, -) - -CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/claude-opus-5-5@default", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-opus-5-5@default:rawPredict", - expected_provider_headers={"authorization": "Bearer scripted-token", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": "Say hello."}], - "max_tokens": 64, - "stream": False, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "vertex-2023-10-16", - }, - mock_provider_response={ - "model": "claude-opus-5-5", - "id": "msg_vrtx_011Cfjuz1zxAEM1YiVFYdaJB", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - }, - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "model": "vertex_ai/claude-opus-5-5@default", - "id": "msg_vrtx_011Cfjuz1zxAEM1YiVFYdaJB", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - }, - }, -) - GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -418,111 +301,285 @@ GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( }, ) -GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( +CLAUDE_SONNET_5_5_MESSAGES_THINKING_SIGNATURE_0: Final = ( + "CAQSiQYKEggSEAIYAjgBQgh0aGlua2luZxIMgrCyOX5aamkI52eqGgy+6kkncswGkygvTJMiMP0m0811aMrw7nEXG8RULCLVjlRy" + "YnR1NVQMPi4D8+0wd7GpOQ3kYb6rRHr4eZQrXiqkBeZyXjb/2wJIq+K8HxoIHFJ1u6xQrkMnZ/tA6M7Kt9Hvftp9EvdEDGYlEN9t" + "aLY0iXPBS0XwePUByV3ZDxTgNx2wXj1K88BwXtpQYYdDUFFb0pC5/HlxxZE1xHeQpHX/7sfTroWGNNco1JKKMC9FfT/YkHM+CaE+" + "LnIaPmL2Ft5t/4UhwBcgIbGIrj0INS0QObLNVSy32zLbxiyaDzC/ZVb9FC+mzRsQdkUpKDsiZMc0quyLrmuCl0cxuypCFiwPXrTM" + "mRSMZiHN6LG/D0w9ySgHalASqB11GuAKPq34XXfK/+xbgwpGnBSKqFYn5CZkVzjV9sRdNMxRd+NjL/RN6DbyFm3xDF9VapDqjkuc" + "MQUUJbHRDP7nIVRPGvSqE1cE2qhrimR8uKKH9MtjMG6EPAaL3gMWyIwpxM7UOX602XYgpQpxNa2yGdtv2R8740FrKRucYsKkCRbo" + "fO7SI8p+mR8KChm0zY6Q+qFWsZvaeQZPrxvERghOKvXDBwbCtR2OIru2ru6JeBfNqJZsIsPGBBENEjI0LkB/D9/iURQaqO1wgXLb" + "HWtSt1ErxTtGRjWEKCEETfy77eoWptmgGzitX8Qoim25TVjTK2Pig7rWfapiOrxpLFo5zGfDuKJuVqDdcS77BXQ+LpfzaD0AWoOx" + "z3VCLyX0wgttyDDuAF423Iu6HESf0z6FbPpZNiuJaAARNZAFYhp0AFHCCwyRtGqUuas75Jzp2bpCWzlpzJVXZ9vbWS+AYkBEwaig" + "MuGS6q5bz5efcypwiv2+OHPGOh7Wj73BgGyiwNg8KLGIR/xWEkGuNRPrApty7ABzOlZNIK9Ti1Ay2yCBIZWQSAfTPmUBcZi5xlgB" + "LmA+bljzJJq/cJ4P7wec/5WeHyYVEwQ5pCOcS3mRj5QYAQ==" +) + +CLAUDE_SONNET_5_5_MESSAGES_THINKING_SIGNATURE_1: Final = ( + "CAQS3QUKEwgSEAIYAjgBQgluYXJyYXRpb24SDBELWnnn6q16EbJTkBoM4dA0bbJp1Z9s2epTIjALivb2lWtreb6CmIP6jl8Af48M" + "XpiMfs5leblsmVBauYg0j+JPBYYDD92ttpEDBiwq9wSfdfC1d1MhZgmJ+dB/DAUva2swtVmdUIpxDfPT2jXA1v/zydQ+GUGGwfGX" + "FoaACIyUPYGR1nmtfrxp19wWKJkFFkw/NM+pjmK+yTLlExT3ptqwPmmCTC5q8Zh6GjWnKi0OXVdo2fkb/4OhkX8Xpib1TLL8iwD8" + "7+jwqNbbNMIE+V5Pf96dI0yCkFFWNmtMf538o3rfsk1lvD7J1XR2ISOur0P0osPbG0huA/2oZrgLw5ZEWI+myvStm7JTnP31Jc+N" + "0JJQDhhTZOaWqxPmzr2AB83eaLSWqpgBnuPnpzgMP0Q3qkQjGmGFO35FkvN0QiTDgsQXxqVT+seLM5KJ17h+H2+Gj16atqfD2cD9" + "WzDrEAPxD48srrtbtcIrWkz9PuaezfqRanwj1HZWL8CKDPxkvASw3v2vwHYfCZ772KckH/xElpoa4HSLt8/MJqpNklxEDV6MIQgM" + "y0gkfq4NlqRff2+yUsLEQdEQowX6EjFhjcvDZEJSHGoA5cNWBl0rL7GIoFOlDtqLo6yU9JzKQrhVmaST9xKSqTbQ2UlMTZaAWc51" + "dEyf5APmS+o/xcwnMa3T4l1NWhtZgVCBv9L2Eq7FxP+VNeaIaaY+GJw0EtKo5zEsik39uBbFhzCIIMZfcjN+EAFXy0y6/AUiQjlj" + "BSiI14q3AIhGxn9zMuPiPTMdbL+UNHNSTDOccJZJAR6P07T+5TSFtnXQhau7nooQr1cirtLYixb8KeYZRTqq4CAUXdI5gfq6P6jt" + "3oJm5yqmTJ/gKn46uv3mtWMVKnKmflfLT8bmYcm5ZVtyN+T6g9zwraZpdmA11u/GszOiJkglV1Vr3MyTA9CNGAE=" +) + +CLAUDE_OPUS_5_5_MESSAGES_THINKING_SIGNATURE_0: Final = ( + "CAQSuAUKEggSEAIYAjgBQgh0aGlua2luZxIMuvOumb0X7WEvCG/pGgyEiubGiCWEMWg7YgQiMOhK4Mj8OKMMZOczi5roOTadgSLN" + "L6MOk46J66LvPfihwIwowsxxkYa+EszhqUBYXSrTBIe+i63dRgUJd70m96/EnLB3GHAq+Q0ZhpE/USuwpJTc+MvI7KpuGtExRfnk" + "OXzocJOw+R6FxqDO2dHHp00t9+I1gKCFUTqrSfKO0YZspMYfJGzSRjx5vx1equok7iV0k0CoQ8kMpRLhD1zTvf862EX8GpMcNkQs" + "unrK/y7sQAhjPEDtSW4V8ipJF6E6RfJv5YOEsQKVx1kugmQEmItq4TLSMZd06Cip5CIkYrSsGhLYBfv5S2PFjXQGSOSkG9MDqjbI" + "kPzA/1H38oIpdRjvQYMnrfAo+SaXuXybcUM36Svtj/+KCfG0A7Qb+Xhs2fT322lCpneyN1hXpqu6C0xGipHn2UDmIKxXcuOcet6J" + "cCsjUB5BGf1H28hTtqZxunXUpZ9amTkDnyiXEEd7cwcdvFaqVd3/vRPJMSHLwbL8zY0VkWProBV54vW8GgPCFVWFw9B6LxjoPpbn" + "7DiDlPPpQMik03iC3z2K3HPunID3Zx5LQ80nhueu0pec6rmWKijzQy/KVP5YRd8e2w2WpEqi8OwbJzFqUC7Eb8YIpjSs629/vhW2" + "xJt09F0z2sbrs6JS8XZf0zzxWWLjoToWw/YKoJ/GOnJKxWJNI/4FutZ2Z79Xtdtk40DVQbjD4HT6MTw4r2z2UbnwkvACfUhqc5yE" + "kH4lsmx5tSzh1kLfJEAmqwYrHDwYCvT3AUrl9f0EMPQiZpE0yPu1A52y4Ko/6uQWGHddGqtG96kLqb8BhPQR4p+XapjW/6MW6gmn" + "kqrPPK6cl3CzcCy83Qojr1P3tt4CHu+bOeIYAQ==" +) + +GEMINI_3_8_FLASH_MESSAGES_THOUGHT_SIGNATURE_0: Final = ( + "AY89a1++F/LsezPdJfrsXn9QhCwRsfuV/aT9QBFOa7bhIIgH0w67lP4bEkQBbUUZZ0V8XvQ3oFv8ZHuqJkVSGzArEgbhB2T5eu1o" + "2zdg49SAfe/ys8JU77vwHhY630gFJq9fMFA4Eq60R97uaHZlDS2xp0chq32FIHizcI8fRVpRQf9i77gtdZ2UTaQ27C4ak0XzAsF8" + "eYK5XWPD9mfJUj3qxwl0jJ3jVRo9ooqVHdkcRI6HXFazfuyDex3e/jd7AB9aeEQ7Ju4a1805jcm6UL7IUv1CPeGTuKnW5g/jFMho" + "I5V8Zx2wLmP/IJimlPPvgwtyZ7CqnEEgiViLUUQoBZ9e2RlFP6SyfNs8QsyN+VjGklMM9mTZAGwVPbX2JofYNLoOXVpPQgEmH+lh" + "PjomUTbYyuSMTrWZLBv+QpDAOs/oQaUP0SEX2V9aJHNqB/wLDN/4WygO0Aj3WRmXmJLOiwj8vl2asqtDNW4A2oePoZI4sI2Fh4uz" + "6R/RqBfl4tTKEkAKs+3Eo3geZKaey8bNht+8abaudshgOhbNH9ON7YwM/S18LAcUXmcDWi3L6URw2BpJ/hmUIddsFeSNDJyW9sPj" + "xGmLwMENxL2ETb/EsEnzgFfNMXmdE8JA7eWvFtc8E/TF1qO8+Z/nA9fOi3wtExL0i8TTfIuGPNaYvzS42c+9ZBtIRUa5UqFeJwXa" + "NfHBKXZV+bsIvCLYFGHiaKReTP13KlgU2/pXCH608xM9t5F8Jqs3RSHnK3p4aDCH+9XS0pZd6j+FSiGSlDRf5xCPBV6S8ZE53547" + "z7GWYJBeOGl61q/eEnMZ0PcGczGocHVLaIH0tNtlmSlploCdgdOCbiKH/k4ozDq8+XLhxS/LOfwOS+a6YwoZzEEsxXjwVy05h+qZ" + "QtPU3EirPC38+Hky2bY7ZWh1Kp2PMBECnclLvTlN82rWAxfKV7693pLIAx7cyNy3aC5fU2aR0ByyB215evU8jpoe6LAnBKWdnRab" + "vf4tvp6mZofYzFI+GWlCR/ZvH7rduHhm8L79EP0l6jgXSvkOX+kTcMj4gXB2Ty4SEZB8GfuMBhQeRrnIw5GZXl6llMrdF2fBshv5" + "lEE/xJUvYZVCO6KiUXDz8eSoe3IpxbMPoL7zP0HEaz9jEKIo2zicb9tc2rNptkpBO8lq3XGsgsRS0OaF2HQhyDjmhfAeO3rv1o1o" + "MwcHPRRL74cd8NhgT8u8Ub00tYb8u1xcgXRDcMKY/09/QMxvf0Fbp5wrAf2Mklc5ZGd1dqsPR45iFuI7Oq3yLsufEZ8Y51R0pe+p" + "WjUskLdSMnbVdWhAzSevKY/0MQUhgx5BGkuguWWwsVrllJ+kGus++eDy4ti4Aowlc78brDFJ2S3ooC2h0OBh86w7725JPe/FL+q8" + "ZkbGLlIwqr6YLEqb+xXsBO9ffwbg5LisXLD1+ZNtcEaecrEi2vcRVCKur5Kp0Wk=" +) + +GEMINI_3_1_PRO_PREVIEW_MESSAGES_THOUGHT_SIGNATURE_0: Final = ( + "AY89a1+Kk+qLPv2hT8v72PWRRnf/jCTgUnIw5y2JAFmJUzzsseqTRQA/L/XoGa+l56wDzylZtjpWPupaluVI2ih517CV2zWrLqCU" + "JM/82tw2mp7qOvywPkjUuZL9Pe5qVnCFjQZYbZQIByZY0z75bq71b2gJigTfOQhsD/0LkvZUDUeyyihh1v/uhT33aUVwLCtXrJ1b" + "vkWS8CG204JzwgFKAXzYoNV99iro/PN7REisc0rc3osleu93uRExX028ahy9vCvzXh8XFfgjMBTj+5O0clsl5EsF2wSnRiB3llfh" + "ETYs+5v6VMHpYPXpg9M0T/S0UltgN8jDUxKVrroot7bjDHGDA1przgemEk4iJ/kkf3ZgEQS79Hyn/cF6OLnXuQikr3Mo3qYMS+S+" + "sm9E62/xoPCuzP7vUAg8uzYzpNmqKECV+mx7bIb2jRe8fYEsE0ctZ1sc4mxjsXqLKFz/ZqGDraRpxVAp9YXNk4CVtNW7HsvF2yUC" + "7E0Z68mR0AvUuIAizHHAasYgOAGVDYJrwTvR2cr7HQcxz1JoHWe9N09y1AOJI8qnaESryZo1NWatTgb/4ei32Zp4eB71u0S8Xq/I" + "bRPfreLSDZe+UhNM0Qy2vscUZd6iR1NO0JY03GWxJFzSdjDJi2s3qJEIgtlWNayUIl1NbIzTDlCV2Ar1e5b9U4nJLvRb2g9dJPGP" + "eFiNg6/wLaaNvgtxXqJNoYR8gFB2rY5NHBm23Po00vPsnb3qW7AEkOTSTc+3W5krqAsIWLYo+A17hs+kHaj1Q5kuQ0JAdZQHSLYe" + "SJMvfMR/Cq06i+2sfXVbJYBnDqLrNXTBKY7zlat2S/Ku32s1qyi2dDnNxKNFbE7/z5xPJNSQE+icVG8dKKXKi9k9H5utqJQPuDc/" + "mHYWCccrrEZa1A18E5TVVLpBlCeHILz80FB50WbJ8FZP2kUAzR1XlMWI/N6Td4+n1Ewl7kNneT9cBXtEoXZHyjNOCjEeKrZjB8C4" + "LK6Ck6NmG4+1InP7dlVxiZ1uVKnnp7U2eXGNS4zU6fZ1wQbG37cAiqbXj/76f6ez4/w=" +) + +GEMINI_4_ARGON_MESSAGES_THOUGHT_SIGNATURE_0: Final = ( + "AY89a1/RSn20UE3dvcqrPsNoEs739i6LfLmFV/lDmMH140T7bWArO7hY0MDQce+DRbIBGU+d1ZkqSakUkfHGg2OQ3gNRavTI64N4" + "7kw/3NpbO57RZq7o8VfnE4AqsFw+LnquTYD1JO+kT68ovHJXc+drutL5R32m5o0wu0gCxq2zAlc4qjiXWKWF+HTzaCGqx7xRygUD" + "DPiPAodLqLhTNFNSSPs1zNDHALznl6CNHOZa2gNv8yN66BWn9+wHbt7WuEpACcQGJpfMjqHBoDDb0alSRNgO6r3+m1yIKQ+isZIJ" + "MMybfP6dV2erAKPuoYC+YJHgAK0vi7K9aMJwySFB9syBoF2s6iBg20fZXomBn+iF1/W+vEw+Tf3wJ//WK9qnKkNkKk9OeKk3bN2B" + "D7jEahual5cQzx5RPiDxB5DLPfxr5J+XZvXP3ZmB9V272LBT8sIPdmuc2kMt/1drXirh+z6ia8V/ePmOKcgyCBKSogFMWGFTydBh" + "1sNz79aWiofiXlC0JSovAveQ+46nDm7D3bXPA2w0umz/ac9v/95suuGDq4ijo840DqqLLGXUu/ZSyUb++LqvSkg5YYdvc6sQQmPd" + "ztygGH0dC6udIKvCuOYo+0ctUN/9HGSUUvjTBYT0kv7gXtrqe8cV7MPasGKHWYpaHspVu0k0qXQo4PdINS4lnetIIT+lV9L6sxvk" + "KVpmjeE3ktitlE3GyjhyznzcB0DCNpGlWf13GXBokyv4OMtr6I+jYtE2TZRsSTBv6wEmNwnB50yWvBPSph86Qz5Yf4S68KQBEbwH" + "bssQoZ3DgoG2sk+ybNbAEsBLCU+35/8iMk87/dngk7mbM+EaVaboyGBm9KXzh8Kk" +) + + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/gemini-3.8-flash", - "max_tokens": 1024, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.8-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, + litellm_request={**MESSAGES_REQUEST, "model": "vertex_ai/claude-sonnet-5-5@default"}, + expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-sonnet-5-5@default:rawPredict", + expected_provider_headers={"authorization": "Bearer scripted-token", "content-type": "application/json"}, expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, + **VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + "messages": MESSAGES_REQUEST["messages"], + "stream": False, + "system": [ + { + "type": "text", + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + "tools": MESSAGES_REQUEST["tools"], }, mock_provider_response={ - "candidates": [ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-sonnet-5-5", + "id": "msg_vrtx_011Cg38paXLscBt8ARDQWjmd", + "content": [ { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } + "type": "thinking", + "thinking": ( + "Since the user asked for the validation file first, I'll focus there rather than reading tests in " + "parallel. I'll pull a range around line 18, say lines 1-60, to get enough context.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_MESSAGES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "Next I'll check `shop/validation.py` to see how `validate_quantity` handles zero, negative, and " + "non-integer values, starting from the top of the file around line 18.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_MESSAGES_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01UHVgrFsbouYKZ8mjVaM2wj", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 118, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 107, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 257, + "output_tokens_details": {"thinking_tokens": 159}, }, - "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-05T22:55:12.739855Z", - "responseId": "0CrEao-ULbuU9LsPk-mQ2A8", }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": "0CrEao-ULbuU9LsPk-mQ2A8", - "type": "message", - "role": "assistant", - "model": "vertex_ai/gemini-3.8-flash", - "stop_sequence": None, - "usage": {"input_tokens": 9, "output_tokens": 109}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, + "model": "vertex_ai/claude-sonnet-5-5@default", + "id": "msg_vrtx_011Cg38paXLscBt8ARDQWjmd", + "content": [ + { + "type": "thinking", + "thinking": ( + "Since the user asked for the validation file first, I'll focus there rather than reading tests in " + "parallel. I'll pull a range around line 18, say lines 1-60, to get enough context.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_MESSAGES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "Next I'll check `shop/validation.py` to see how `validate_quantity` handles zero, negative, and " + "non-integer values, starting from the top of the file around line 18.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_MESSAGES_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01UHVgrFsbouYKZ8mjVaM2wj", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 257, + "output_tokens_details": {"thinking_tokens": 159}, + }, }, ) -GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/gemini-3.1-pro-preview", - "max_tokens": 1024, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.1-pro-preview:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, + litellm_request={**MESSAGES_REQUEST, "model": "vertex_ai/claude-opus-5-5@default"}, + expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-opus-5-5@default:rawPredict", + expected_provider_headers={"authorization": "Bearer scripted-token", "content-type": "application/json"}, expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, + **VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + "messages": MESSAGES_REQUEST["messages"], + "stream": False, + "system": [ + { + "type": "text", + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + "tools": MESSAGES_REQUEST["tools"], }, mock_provider_response={ - "candidates": [ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-opus-5-5", + "id": "msg_vrtx_011Cg38pLy1pX8GQEfjq6NFo", + "content": [ { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } + "type": "thinking", + "thinking": ( + 'Since the user said "first," I\'ll focus on reading the validation file around line 18 before moving ' + "on to the test file.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_MESSAGES_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": ( + "I'll start with `shop/validation.py`. `validate_quantity` is defined at line 18, so I'll read the " + "lines around it. That should show how zero, negative and non-integer quantities are handled." + ), + }, + { + "type": "tool_use", + "id": "toolu_vrtx_018jLLjmUz5DQCg8xZ76kMFY", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + }, ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 1, - "totalTokenCount": 99, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 1}], - "thoughtsTokenCount": 89, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 233, + "output_tokens_details": {"thinking_tokens": 70}, }, - "modelVersion": "gemini-3.1-pro-preview", - "createTime": "2026-10-05T22:55:14.874538Z", - "responseId": "0irEaqqwNbCdq8YP3L2mqQ4", }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": "0irEaqqwNbCdq8YP3L2mqQ4", - "type": "message", - "role": "assistant", - "model": "vertex_ai/gemini-3.1-pro-preview", - "stop_sequence": None, - "usage": {"input_tokens": 9, "output_tokens": 90}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, + "model": "vertex_ai/claude-opus-5-5@default", + "id": "msg_vrtx_011Cg38pLy1pX8GQEfjq6NFo", + "content": [ + { + "type": "thinking", + "thinking": ( + 'Since the user said "first," I\'ll focus on reading the validation file around line 18 before moving ' + "on to the test file.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_MESSAGES_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": ( + "I'll start with `shop/validation.py`. `validate_quantity` is defined at line 18, so I'll read the " + "lines around it. That should show how zero, negative and non-integer quantities are handled." + ), + }, + { + "type": "tool_use", + "id": "toolu_vrtx_018jLLjmUz5DQCg8xZ76kMFY", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 233, + "output_tokens_details": {"thinking_tokens": 70}, + }, }, ) @@ -530,221 +587,448 @@ GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/claude-fable-5-1@default", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "vertex_ai/claude-fable-5-1@default"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-fable-5-1@default:rawPredict", expected_provider_headers={"authorization": "Bearer scripted-token", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": "Say hello."}], - "max_tokens": 64, + **VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + "messages": MESSAGES_REQUEST["messages"], "stream": False, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "vertex-2023-10-16", + "system": [ + { + "type": "text", + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + "tools": MESSAGES_REQUEST["tools"], }, mock_provider_response={ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_vrtx_011Cg33y2uVUY4EWp5v3Eh7T", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_vrtx_011Cg38pBu4rsBNuTwSrvoze", + "content": [ + { + "type": "text", + "text": "I'll read the validation function to see how zero, negative, and non-integer quantities are handled.", + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01SNoqWEthDpRET4kp9iGsXB", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 129, "output_tokens_details": {"thinking_tokens": 0}, }, }, - expected_litellm_status_code=200, expected_litellm_response={ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "vertex_ai/claude-fable-5-1@default", - "id": "msg_vrtx_011Cg33y2uVUY4EWp5v3Eh7T", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_vrtx_011Cg38pBu4rsBNuTwSrvoze", + "content": [ + { + "type": "text", + "text": "I'll read the validation function to see how zero, negative, and non-integer quantities are handled.", + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01SNoqWEthDpRET4kp9iGsXB", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 129, "output_tokens_details": {"thinking_tokens": 0}, }, }, ) -GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/gemini-3.5-flash", - "max_tokens": 1024, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.5-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_5_FLASH_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 95, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 84, - }, - "modelVersion": "gemini-3.5-flash", - "createTime": "2026-10-05T22:54:50.809002Z", - "responseId": "uirEaqqwMc-U9LsPjJTYuQU", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": "uirEaqqwMc-U9LsPjJTYuQU", - "type": "message", - "role": "assistant", - "model": "vertex_ai/gemini-3.5-flash", - "stop_sequence": None, - "usage": {"input_tokens": 9, "output_tokens": 86}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, - }, -) GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/gemini-3.8-flash", - "max_tokens": 1024, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "vertex_ai/gemini-3.8-flash"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.8-flash:generateContent", expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, + **VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + }, }, mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, "candidates": [ { "content": { "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], + "parts": [ + { + "text": ( + "**My Thought Process on Reviewing Quantity Validation**\n\nMy primary objective is to understand how " + "the system handles zero, negative, and non-integer quantities, as this is crucial for ensuring the " + "integrity of our synthetic Python project. The user's initial request directly guides me to the " + '"validation file first so we can understand how zero and negative quantities are handled."\n\nIn ' + "alignment with my system instructions, I must first identify the next relevant file, explain why " + "I'm choosing it, and then use a read-only tool to inspect it. I also need to explicitly keep in " + "mind the nuances of zero, negative, and non-integer quantities as I proceed.\n\nTherefore, the " + "immediate next step is to examine the validation logic. Based on this, I've deduced that " + "`shop/validation.py` is the file I need to inspect. This file is likely to contain the core logic " + "for validating quantities, and I anticipate it will provide the answers regarding how these " + "specific edge cases (zero, negative, and non-integer values) are managed.\n\nTo get a comprehensive " + "view of this validation mechanism, I will focus on the initial section of the file, particularly " + "around the function `validate_quantity(quantity)`, which I recall is located at line 18. I'll " + "request to read lines 1 through 40 to get a good understanding of the surrounding context and the " + "validation function itself.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I need to read `shop/validation.py` next to see how `validate_quantity` validates incoming quantity " + "values, including checks for zero, negative, and non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"end_line": 40, "path": "shop/validation.py", "start_line": 1}, + "id": "call_190093", + }, + "thoughtSignature": GEMINI_3_8_FLASH_MESSAGES_THOUGHT_SIGNATURE_0, + }, + ], }, "finishReason": "STOP", - } + }, ], "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 118, + "promptTokenCount": 242, + "candidatesTokenCount": 71, + "totalTokenCount": 556, "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 107, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 242}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 71}], + "thoughtsTokenCount": 243, }, "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-05T22:55:12.739855Z", - "responseId": "0CrEao-ULbuU9LsPk-mQ2A8", + "createTime": "2026-10-15T01:13:45.750641Z", + "responseId": "ySjQarHoLb2apucPwee_YQ", }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": "0CrEao-ULbuU9LsPk-mQ2A8", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "ySjQarHoLb2apucPwee_YQ", "model": "vertex_ai/gemini-3.8-flash", - "stop_sequence": None, - "usage": {"input_tokens": 9, "output_tokens": 109}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 242, "output_tokens": 314}, + "content": [ + { + "type": "thinking", + "thinking": ( + "**My Thought Process on Reviewing Quantity Validation**\n\nMy primary objective is to understand how " + "the system handles zero, negative, and non-integer quantities, as this is crucial for ensuring the " + "integrity of our synthetic Python project. The user's initial request directly guides me to the " + '"validation file first so we can understand how zero and negative quantities are handled."\n\nIn ' + "alignment with my system instructions, I must first identify the next relevant file, explain why " + "I'm choosing it, and then use a read-only tool to inspect it. I also need to explicitly keep in " + "mind the nuances of zero, negative, and non-integer quantities as I proceed.\n\nTherefore, the " + "immediate next step is to examine the validation logic. Based on this, I've deduced that " + "`shop/validation.py` is the file I need to inspect. This file is likely to contain the core logic " + "for validating quantities, and I anticipate it will provide the answers regarding how these " + "specific edge cases (zero, negative, and non-integer values) are managed.\n\nTo get a comprehensive " + "view of this validation mechanism, I will focus on the initial section of the file, particularly " + "around the function `validate_quantity(quantity)`, which I recall is located at line 18. I'll " + "request to read lines 1 through 40 to get a good understanding of the surrounding context and the " + "validation function itself.\n\n" + ), + "signature": "", + }, + { + "type": "text", + "text": ( + "I need to read `shop/validation.py` next to see how `validate_quantity` validates incoming quantity " + "values, including checks for zero, negative, and non-integer quantities." + ), + }, + { + "type": "tool_use", + "id": "call_190093", + "name": "read_file", + "input": {"end_line": 40, "path": "shop/validation.py", "start_line": 1}, + "provider_specific_fields": {"signature": GEMINI_3_8_FLASH_MESSAGES_THOUGHT_SIGNATURE_0}, + }, + ], }, ) + GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "vertex_ai/gemini-3.1-pro-preview", - "max_tokens": 1024, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "vertex_ai/gemini-3.1-pro-preview"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.1-pro-preview:generateContent", expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, + **VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + }, }, mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, "candidates": [ { "content": { "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], + "parts": [ + { + "text": ( + "**My Thought Process on Quantities in the Shop Module**\n\nMy immediate goal here is to gain a " + "precise understanding of how the `shop` module, specifically within its `validation.py` file, " + "handles the input of item quantities, with a particular focus on zero and negative values. This is " + "a critical area for ensuring data integrity and preventing potential issues downstream in any " + "e-commerce or inventory system.\n\nTo achieve this, I need to examine the source code directly. I've " + "identified that the relevant logic for quantity validation is likely contained within a function " + "named `validate_quantity`. Based on my indexing or prior knowledge of this codebase, I know this " + "function resides in `shop/validation.py`. I'll proceed by reading a section of this file that " + "encompasses the `validate_quantity` function. I'm aiming for a contextually rich view, so I'll " + "fetch lines from around where I expect the function to start. My tool call will be to `read_file`, " + 'specifying the path as `"shop/validation.py"`. To ensure I capture the function definition and its ' + "immediate surrounding logic, I'll set `start_line` to `1` and `end_line` to `30`. This range should " + "provide me with sufficient context to analyze the function's implementation.\n\nOnce I have the " + "content of `shop/validation.py` for those lines, I will meticulously review the `validate_quantity` " + "function to observe how it explicitly checks for and handles zero and negative quantity inputs. " + "This will involve looking for conditional statements (`if`, `elif`, `else`) and any associated " + "error-raising mechanisms or default behaviors.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I will read the `shop/validation.py` file to see how `validate_quantity` handles zero and negative " + "quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"start_line": 1, "end_line": 35, "path": "shop/validation.py"}, + "id": "call_1455553", + }, + "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_MESSAGES_THOUGHT_SIGNATURE_0, + }, + ], }, "finishReason": "STOP", - } + }, ], "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 1, - "totalTokenCount": 99, + "promptTokenCount": 242, + "candidatesTokenCount": 59, + "totalTokenCount": 499, "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 1}], - "thoughtsTokenCount": 89, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 242}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 59}], + "thoughtsTokenCount": 198, }, "modelVersion": "gemini-3.1-pro-preview", - "createTime": "2026-10-05T22:55:14.874538Z", - "responseId": "0irEaqqwNbCdq8YP3L2mqQ4", + "createTime": "2026-10-15T01:13:40.675781Z", + "responseId": "xCjQasWfKcyL9LsPx7HAqQ8", }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": "0irEaqqwNbCdq8YP3L2mqQ4", - "type": "message", - "role": "assistant", + **MESSAGES_BRIDGE_RESPONSE, + "id": "xCjQasWfKcyL9LsPx7HAqQ8", "model": "vertex_ai/gemini-3.1-pro-preview", - "stop_sequence": None, - "usage": {"input_tokens": 9, "output_tokens": 90}, - "content": [{"type": "text", "text": "Hello."}], - "stop_reason": "end_turn", - "stop_details": None, + "usage": {"input_tokens": 242, "output_tokens": 257}, + "content": [ + { + "type": "thinking", + "thinking": ( + "**My Thought Process on Quantities in the Shop Module**\n\nMy immediate goal here is to gain a " + "precise understanding of how the `shop` module, specifically within its `validation.py` file, " + "handles the input of item quantities, with a particular focus on zero and negative values. This is " + "a critical area for ensuring data integrity and preventing potential issues downstream in any " + "e-commerce or inventory system.\n\nTo achieve this, I need to examine the source code directly. I've " + "identified that the relevant logic for quantity validation is likely contained within a function " + "named `validate_quantity`. Based on my indexing or prior knowledge of this codebase, I know this " + "function resides in `shop/validation.py`. I'll proceed by reading a section of this file that " + "encompasses the `validate_quantity` function. I'm aiming for a contextually rich view, so I'll " + "fetch lines from around where I expect the function to start. My tool call will be to `read_file`, " + 'specifying the path as `"shop/validation.py"`. To ensure I capture the function definition and its ' + "immediate surrounding logic, I'll set `start_line` to `1` and `end_line` to `30`. This range should " + "provide me with sufficient context to analyze the function's implementation.\n\nOnce I have the " + "content of `shop/validation.py` for those lines, I will meticulously review the `validate_quantity` " + "function to observe how it explicitly checks for and handles zero and negative quantity inputs. " + "This will involve looking for conditional statements (`if`, `elif`, `else`) and any associated " + "error-raising mechanisms or default behaviors.\n\n" + ), + "signature": "", + }, + { + "type": "text", + "text": ( + "I will read the `shop/validation.py` file to see how `validate_quantity` handles zero and negative " + "quantities." + ), + }, + { + "type": "tool_use", + "id": "call_1455553", + "name": "read_file", + "input": {"start_line": 1, "end_line": 35, "path": "shop/validation.py"}, + "provider_specific_fields": {"signature": GEMINI_3_1_PRO_PREVIEW_MESSAGES_THOUGHT_SIGNATURE_0}, + }, + ], + }, +) + + +GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "vertex_ai/gemini-4-argon"}, + expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-4-argon:generateContent", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, + expected_provider_request={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the messages " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + }, + }, + mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": ( + "**My Thought Process on Handling Quantity Validation**\n\nMy primary objective is to meticulously " + "understand how the system currently manages the validation of product quantities, specifically " + "focusing on how it addresses zero and negative values. To achieve this, I need to first dive into " + "the `shop/validation.py` file. This will provide me with the foundational knowledge of the existing " + "logic and the rules that govern quantity inputs.\n\nFollowing the system's instructions, my next step " + "is to explicitly state which file I require for this understanding and then invoke the `read_file` " + "tool to access `shop/validation.py`. I must also keep in mind the overarching directive to consider " + "zero, negative, and non-integer quantities throughout this process, as this will be crucial when I " + "eventually propose any solutions or fixes. It's important to remember that I am not to alter any " + "files, only to read and analyze them, and to communicate my progress and needs via the provided " + "messages client.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I will read `shop/validation.py` next to inspect how `validate_quantity` handles zero, negative, " + "and non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"end_line": 50, "start_line": 1, "path": "shop/validation.py"}, + "id": "call_258575", + }, + "thoughtSignature": GEMINI_4_ARGON_MESSAGES_THOUGHT_SIGNATURE_0, + }, + ], + }, + "finishReason": "STOP", + }, + ], + "usageMetadata": { + "promptTokenCount": 412, + "candidatesTokenCount": 69, + "totalTokenCount": 604, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 69}], + "thoughtsTokenCount": 123, + }, + "modelVersion": "gemini-4-argon", + "createTime": "2026-10-15T01:13:52.985186Z", + "responseId": "0CjQauKQPNyFpucPldSPiAY", + }, + expected_litellm_response={ + **MESSAGES_BRIDGE_RESPONSE, + "id": "0CjQauKQPNyFpucPldSPiAY", + "model": "vertex_ai/gemini-4-argon", + "usage": {"input_tokens": 412, "output_tokens": 192}, + "content": [ + { + "type": "thinking", + "thinking": ( + "**My Thought Process on Handling Quantity Validation**\n\nMy primary objective is to meticulously " + "understand how the system currently manages the validation of product quantities, specifically " + "focusing on how it addresses zero and negative values. To achieve this, I need to first dive into " + "the `shop/validation.py` file. This will provide me with the foundational knowledge of the existing " + "logic and the rules that govern quantity inputs.\n\nFollowing the system's instructions, my next step " + "is to explicitly state which file I require for this understanding and then invoke the `read_file` " + "tool to access `shop/validation.py`. I must also keep in mind the overarching directive to consider " + "zero, negative, and non-integer quantities throughout this process, as this will be crucial when I " + "eventually propose any solutions or fixes. It's important to remember that I am not to alter any " + "files, only to read and analyze them, and to communicate my progress and needs via the provided " + "messages client.\n\n" + ), + "signature": "", + }, + { + "type": "text", + "text": ( + "I will read `shop/validation.py` next to inspect how `validate_quantity` handles zero, negative, " + "and non-integer quantities." + ), + }, + { + "type": "tool_use", + "id": "call_258575", + "name": "read_file", + "input": {"end_line": 50, "start_line": 1, "path": "shop/validation.py"}, + "provider_specific_fields": {"signature": GEMINI_4_ARGON_MESSAGES_THOUGHT_SIGNATURE_0}, + }, + ], }, ) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py index 218bb8069c2..d423c9465ec 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_anthropic.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/messages/basic/test_messages_basic_azure.py b/tests/integration/translation/messages/basic/test_messages_basic_azure.py index 2f76ccc3028..7d84f7be0e7 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_azure.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_azure.py @@ -6,6 +6,7 @@ from integration.translation.messages.bases.azure import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -22,6 +23,7 @@ from integration.translation.runner import assert_translation GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_azure_ai.py b/tests/integration/translation/messages/basic/test_messages_basic_azure_ai.py index cbaa5a4536e..1303aea4788 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_azure_ai.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_azure_ai.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py index e10fec66852..7cf199ee276 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_converse.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_invoke.py b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_invoke.py index 0ee60ee0e74..d387f318322 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_invoke.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_invoke.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_mantle.py b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_mantle.py index 80a55d9eb35..e00d9694ff5 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_mantle.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_mantle.py @@ -9,7 +9,9 @@ from integration.translation.messages.bases.bedrock_mantle import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, + GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, ) from integration.translation.runner import assert_translation @@ -26,6 +28,8 @@ from integration.translation.runner import assert_translation GPT_5_6_LUNA_TEST_CASE, GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, + GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_gemini.py b/tests/integration/translation/messages/basic/test_messages_basic_gemini.py index c217ce39478..93f5809de8c 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_gemini.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_gemini.py @@ -6,13 +6,19 @@ from integration.translation.messages.bases.gemini import ( GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ) from integration.translation.runner import assert_translation @pytest.mark.parametrize( "case", - [GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE], + [ + GEMINI_3_5_FLASH_TEST_CASE, + GEMINI_3_8_FLASH_TEST_CASE, + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, + ], ids=lambda case: case.id, ) def test_messages_basic_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: diff --git a/tests/integration/translation/messages/basic/test_messages_basic_openai.py b/tests/integration/translation/messages/basic/test_messages_basic_openai.py index d96308e9d92..38d864411c0 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_openai.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_openai.py @@ -6,6 +6,7 @@ from integration.translation.messages.bases.openai import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -22,6 +23,7 @@ from integration.translation.runner import assert_translation GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_vertex_ai.py b/tests/integration/translation/messages/basic/test_messages_basic_vertex_ai.py index 14bfda9bb67..a5b01dfd2e4 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_vertex_ai.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_vertex_ai.py @@ -1,19 +1,19 @@ import pytest - from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase from integration.translation.messages.bases.vertex_ai import ( CLAUDE_FABLE_5_1_TEST_CASE, CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_SONNET_4_6_TEST_CASE, - CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, - CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, - GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ) from integration.translation.runner import assert_translation @@ -31,6 +31,7 @@ from integration.translation.runner import assert_translation GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_azure_ai.py b/tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_azure_ai.py new file mode 100644 index 00000000000..2f32382e9cf --- /dev/null +++ b/tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_azure_ai.py @@ -0,0 +1,126 @@ +from collections.abc import Mapping +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.azure_ai import CLAUDE_FABLE_5_1_TEST_CASE, CLAUDE_HAIKU_5_5_TEST_CASE +from integration.translation.runner import assert_translation +from pydantic import JsonValue + +CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0: Final = ( + "CAQS6gUKEAgSGAI4AUIIdGhpbmtpbmcSDLQu922okJauVttVBBoMAgcnYaXPP2pLp7kWIjBjWY/rKcIkRW/cK1zER1RvizqI4Z+33qxyJw9F" + "e+n1/BHcFPZB/YEHmGpyettyI5QqhwUoNUlt00bsx60ekxYVjn/ocAkKHVywtpoVpn65SrGRBSzMIfVoEtecE/05Gzvy1GgO8L1995thSB+M" + "E9+Lzc+bvIdirQ/jLWnlnE1L/Vg6/YtM4eUy8a4c8nd//zCPBxgHkZZDrvOKfFE2Md8eKPoB6Uh64DxV0qdcWpPqxdCviDEJzhARb0WMxOWt" + "irKLLOTF+oc546ftUkF5xsbk+qN/96EVdjNwsYWr8OGv+OQdpEpJsT1ANkkverU2iFQ4oNJnW3ORn0uv6uccIRqFuyTFqXN4JefRGUvc8vtV" + "iMdrUr3rIu6j/HCu9Lstm2+SVBnV1wWFzFi8HiIupfOebCGYb5LhvYlWR6UJPk8snpbw9/MfiCqLn+wW/pKvaTSQtuKvQUORftkyVeBmy0Mx" + "UoeSiImk/CqbZ7u7t6wakbFXVZPc5zB1FJe9ymWVCACAFPayHJRLFU93wHjZ2ChPwYhG75HpiA3OordlF5usKP6orO95WU4h8w2FPuX0M2Zz" + "K78MvESEUn/tcf0qthco+y9A1RCPF+NAdgagx3dDXNatT7/4W33yTdf/ZjFFS8UFKJ/vS83Ehp+OhUzVNB4g1g9134JqvUz040Mrctqnibt1" + "+7YKm4nLOHuYxETr5JyHW/noo2X/aNVz8dbL64F6grNe/yofT/D2u7YuaiYnYCwmkQLCK5VmmwExUq43pklgYmgv3X2jaF2f4B6Ph84e3rDV" + "1PQ9AvTgM3WKf0c57NtoMAg4K8NalHtPBELETqI0BnoZt6BW1ltm8ezQoiShdkr8Jy0SpjuaxQmaX9RsoLo1z6LfzaMAf5sXWQFHF8ata5t0" + "bCzIKmeGV5j0+YHZ+wn4HAWu5XjrLhgB" +) + +CLAUDE_HAIKU_5_5_CACHE_HIT_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg37cRp2MtyQ7pW4fdgJX", + "content": [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py, starting with a reasonable initial range to see its " + "contents.\n\n", + "signature": CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "tool_use", + "id": "toolu_01P1gGqhK77Zpmcf2DrLYAUX", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 873, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 167, + "output_tokens_details": {"thinking_tokens": 67}, + "service_tier": "standard", + "inference_geo": "global", + }, +} + +CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_HAIKU_5_5_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_HAIKU_5_5_TEST_CASE.mock_provider_response, + **CLAUDE_HAIKU_5_5_CACHE_HIT_RESPONSE_FIELDS, + }, + expected_litellm_response={ + **CLAUDE_HAIKU_5_5_TEST_CASE.expected_litellm_response, + **CLAUDE_HAIKU_5_5_CACHE_HIT_RESPONSE_FIELDS, + }, +) + +CLAUDE_FABLE_5_1_CACHE_HIT_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg37VFyjKTzJLau9mXGWM", + "content": [ + { + "type": "text", + "text": "I'll read the validation function along with its surrounding context, since the test " + "file also matters but doesn't depend on what the validator says, so I'll fetch both " + "in one go.", + }, + { + "type": "tool_use", + "id": "toolu_01Dr7SroCW5TuKG8UrXY8vad", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_0161J4pCkkCDr4kTDeBdrLFE", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 873, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 248, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "not_available", + }, +} + +CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_FABLE_5_1_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_FABLE_5_1_TEST_CASE.mock_provider_response, + **CLAUDE_FABLE_5_1_CACHE_HIT_RESPONSE_FIELDS, + }, + expected_litellm_response={ + **CLAUDE_FABLE_5_1_TEST_CASE.expected_litellm_response, + **CLAUDE_FABLE_5_1_CACHE_HIT_RESPONSE_FIELDS, + }, +) + + +@pytest.mark.parametrize( + "case", + [CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE, CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE], + ids=[CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE.id, CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE.id], +) +def test_messages_prompt_cache_hit_azure_ai( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/templates.py b/tests/integration/translation/messages/templates.py index 9e8e4a0b053..cb9d59d9d54 100644 --- a/tests/integration/translation/messages/templates.py +++ b/tests/integration/translation/messages/templates.py @@ -82,3 +82,8 @@ MESSAGES_REQUEST: Final[Mapping[str, JsonValue]] = { } MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = ANTHROPIC_MESSAGES_RESPONSE + + +MESSAGES_BRIDGE_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in MESSAGES_RESPONSE.items() if key not in {"container", "diagnostics"} +} diff --git a/tests/integration/translation/responses/bases/anthropic.py b/tests/integration/translation/responses/bases/anthropic.py index b0654b5d24e..e30c1f15699 100644 --- a/tests/integration/translation/responses/bases/anthropic.py +++ b/tests/integration/translation/responses/bases/anthropic.py @@ -1,3 +1,4 @@ +import json from typing import Final from unittest.mock import ANY @@ -105,103 +106,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "anthropic/claude-haiku-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-anthropic-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-haiku-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_011CfoboJTmCCdzhHCYYzodZ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "anthropic/claude-haiku-5-5", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 23, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 23, - "video_tokens": None, - "cache_write_tokens": 0, - }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", @@ -563,16 +467,24 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( expected_litellm_response={**RESPONSES_RESPONSE, "model": "anthropic/claude-sonnet-5-5"}, ) -CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSkgYKEAgSGAI4AUIIdGhpbmtpbmcSDDNkbcUmfOcV76xw1BoM1d74UojfhlwUKXJRIjDruKpmF7Xkn4wU+s/jLhdQL1oTKBrwFivaGL2z" + "2qnyjXgtCgYeYxa4syAiIlaJMkwqrwXgBL0q4FtZvicmkVAfs3GMC/MvUk3zJ5psHQWMfE+EYU436rT6+Xcoj47t9XgRI+Qs5vio51d6AvNR" + "snONcWIVexutCBNKfDOcnBZwzcgr6xlR8+bvZI0a4kgGEkJzqUm5nkST8scZwMkJPsgk0Zrr8aDDcGGmtc161vfYqDIMq6+IpnOlib1Q/FA0" + "gO2Trdo93xLjKvHUXbmSzXokO5x7s60ymFjVd6RzmoLStGgBwiMXElWT40WPh/T2tx2GtuYJ6EPuMOpIqzmI3pLbYplVdVBGdEhUL+wPD8gT" + "mHbBq1Oz8cwSdUyRWBTMCgyZdMkuIw6lPFRAec/19JbDZQWQbnKCOmZK9YwicptzrbllIKqNjJTX4sww8MvMp69lMRxeFqM9tBnC7oIYevEg" + "witiRyKsVfbQTfmexva4g1WenMsqKDub77WJPsbhN0ihuQvlLssmGtRLKh/C4waaaCTzqsj4mDO1UEnflUczRoQHF2fxlsQBVjMc0QcuICYL" + "nwcc2ZL8zvL14sELhiSx2PvxjOIwNuEezF3w3uhS8o/9CSbfJnJL6lQ3lldTA8+WL+cvzN+73+9rTN89GRCJPGni52cMkuGV5ObB2dAyNsBi" + "/WsPwiUetMhWeds/kP+Ct1cqagNuCTenGF+oKf0qNsvIPnWoc8a8YW9bxlEgQ9qBF0NXT26RkEE90Qsp4aseb/qmEDKxE2DMcdMwOru3CEvD" + "xQNy07Dak3+tiSS63kRZwsRFbMTb42W98NrDXqq+sfyvMQarPJj/OmWExt9rna3Y+BlWNq0j82gUyCY4gzv4VHM/OyuhNcmNKDdDsgvSwub9" + "wRSKmuC4LDAhrRMuTQ/cMZq1+ZyQHiuFvnsArZxCusQ5QxDjxRGsI+uMeWmkPY3TN1ofMSIbeWpyiwnsR8EYAQ==" +) + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "anthropic/claude-opus-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "anthropic/claude-haiku-5-5"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", @@ -580,96 +492,308 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={ - "model": "claude-opus-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-haiku-5-5", + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect.", + } + ], }, mock_provider_response={ - "model": "claude-opus-5-5", - "id": "msg_011CfgDzp3URQ38dAAc8F3RP", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-haiku-5-5", + "id": "msg_011Cg3716ZuhZgb2YrqgDQVR", + "content": [ + { + "type": "thinking", + "thinking": "I'll just read the validation.py file directly, starting with a reasonable range " + "near the top of the file since no specific lines were requested.\n" + "\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "tool_use", + "id": "toolu_01H53znxFvBbuHPXNePjEhW5", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 167, + "output_tokens_details": {"thinking_tokens": 67}, "service_tier": "standard", "inference_geo": "global", }, - "diagnostics": None, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "anthropic/claude-opus-5-5", - "object": "response", + **RESPONSES_RESPONSE, + "model": "anthropic/claude-haiku-5-5", "output": [ { - "type": "message", + "type": "reasoning", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "I'll just read the validation.py file directly, starting with a " + "reasonable range near the top of the file since no specific lines were " + "requested.\n" + "\n", + "annotations": [], + }, + ], "phase": None, - } + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "I'll just read the validation.py file directly, starting with a " + "reasonable range near the top of the file since no specific lines were " + "requested.\n" + "\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_01H53znxFvBbuHPXNePjEhW5", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01H53znxFvBbuHPXNePjEhW5", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 167, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 67, "text_tokens": 100}, + "total_tokens": 1044, "cost": None, }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll just read the validation.py file directly, starting with a reasonable " + "range near the top of the file since no specific lines were requested.\n" + "\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + }, }, ) +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSmwYKEAgSGAI4AUIIdGhpbmtpbmcSDAp614h0T9spOzVRohoMRXG5Q7QnUaybHmMQIjDVSvcKksTqmvoyviUgrnx25thBdpJEPfO61S1d" + "hV+zrTe/1FP4+d14zYjlYEZ8Id0quAW5bhrtPXA2ESV73lWwlm/w+SYE+QsiZA9rbSOkSdKLTg37S3i/w+ad4VIFTr1iNoV8RfYZog2D3q69" + "miBOF29U8ILsi+tUM66QNtGdGWDahlsimm+d/IMs+Ie4uP8OHr5X/bJ7NS2MJ407WhS1E+rYrmOce5lHOmhVPGzcL2EM+ToptZ84DR8N7iIq" + "k9pP8CoQL7Z10aCTkG4r2tbdIup7RE3WXrsBHNbs+RAqF5LXbEpsQZaXLx71U1DpDffx6OZ3OSCw1tEcA4o2FDVFNv8CQO3wBnWqkZyjehqb" + "sMarphzyvDxDO7T//sTTVM1593gEbLoRNeUne3bwtgdwoDPMXminh541bLo8vjo25kERKJedEUG/CDZqT16YziXqlSvzV/TgQeddzb2TLnJT" + "LoYoXsmn8789+6nBPdIe3SQu1nhVP5XUgC0+dI/iKqbiuswuslOSx3ZuIEan3i/HZ+XflqXy4ewB0+5sLHbiwzmGEbrZcWl0wbebc2Da0LnY" + "rgAQHqKH0vl4o3fGmaPMxcPml9RP4zG0KblC24o6acb0DYAFTSfblE3dKoMgZwPfiMijqiXriZQSuF5Y3VGY6CcWGQDi5kLzbFe5D2HA/nM4" + "66bjH9VKvVsVglODnnDPQD84e9YM86EelbaxCQcNo9MrCpp7g0tWRvMEiAlfKo0W4hvsYaa2op3XMbHVlJeWJcoU/WPodcEnSEPl6/pz522X" + "I/ItEKXeKIqs0VG9giw0pxuHB8+ig+8bn74Lezmw4LLHk67D2/PCVKLSqrIkHfE41ASdJSFAopc7YY8iovyAVdvUfm+7Vefmmphkx7C162Ze" + "gVLS7mSAxuQCWsIlFyui50D75X8xEa2CdijsbrTvCqKIQHcywq8oLLGmDilMRs1DlnQiAWETjHvpzp3WEs9eZBi4tHPz5CMYAQ==" +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQS0QYKEQgSGAI4AUIJbmFycmF0aW9uEgwL0nQCUkLBdQqyQ48aDBNdOXSFQIfQqDyzeyIwgBeYVWjfsySdPznFBTS6nfeLZiDt+L2EHka6" + "Wp6pH2lO/ZVHd45V/Yy8qr0uX3xrKu0F3QlrjBy505vPXf94O0o7bPvrihxLhHnG9Wf14JgZfAXTqiM7kZxqRCJF2DlP2S4BzspQGIILMu+e" + "gnizwTdKpM7I2XgSqV4ZnQRhjPgRVgQkQrbRfSRtfFWDTcJtz/Lp/2KQw6ijdj1tWc66QB8uNSkTStOlz1sseMaBWNOps0WK+vkJw6J+ZbZK" + "dFdZSgNQ2Iwxl3Pys57YHUrqHaVi3NJVFK760EZeavxciDafSs+ZlTuzeghIy/iLQdSSeUhCl0pZE6SCkEWca+A02GewaHguZ9eRoGnCVsSw" + "soQT3aeK7sUbas1sFDyWSvieJ0naBRUN6ae1QSBAWqYnsvEEx11wWcN/DB3SzjM5j48po9ov+wbeoBl2nex/iUVSfBBixe36nZ8Vt37dC6f+" + "0wgw5DHVtiqFKYSH/b3owUf2RJuYUut7LPkT5qFm0mPlkCqSfZqpRwijVAmV0obs61IgrTVlmFq2igWs0rt9v5iuM7+zjI/DtLL6NgGaFgXW" + "Oc8VjDsrHnUMCK0/QtHuC51aE8mqzw47i4wTQ3O5+/8F8ZJn0PkmxB2lZixCSZ42XhU5pqPiTAWjuXciwXzkCcJKz+CR2cl2gFWw3oQR1N6B" + "f9Wlk3f6DrtPywDFDlY7piBz25k958iZDRvTYkleo8daA+DAWMXS+3k0doIpNBSQMoE67GCgqk3A22huvgYdCqlD142RbrdB7xlUviysajfB" + "lbOFXNjs5Lp+v7UVPiNCDJwha/ZIZfmgp86OSKryKTRrBiMNZRv7v9xF5NyQhB+MzGG3pDzzhqYUlzpIE2vmAd4Hk3/T4X5WvIGW+Juv2U+r" + "pHW59B5K/+2tF3VczG//CpHoB6R+8fNWTDp4Bx+l7v+Pep1lBvVG+XAZZHwaPxeXgaOZcUYs/SMdxYDBflXfvxusttZoXNiXJROZle/lIIeo" + "fDbMvuc14FSkdLJ+N41d4et8zwXI5BcOxPIB7ryDVejwgF/F5kAMtnVdzQEYAQ==" +) + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "anthropic/claude-opus-5-5"}, + expected_provider_endpoint="/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-anthropic-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-opus-5-5", + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect.", + } + ], + }, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-opus-5-5", + "id": "msg_011Cg372KDhhjt1fiWYa1qCb", + "content": [ + { + "type": "thinking", + "thinking": "Since the user asked for the validation file first, I'll start by reading " + "shop/validation.py around line 18 before looking at anything else.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check validate_quantity in shop/validation.py to see how it handles zero, " + "negative, and non-integer inputs before moving to the test file.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_0186wos82AWCLwkPWVeVegg8", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 228, + "output_tokens_details": {"thinking_tokens": 130}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **RESPONSES_RESPONSE, + "model": "anthropic/claude-opus-5-5", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Since the user asked for the validation file first, I'll start by reading " + "shop/validation.py around line 18 before looking at anything else.\n" + "\n" + "I'll check validate_quantity in shop/validation.py to see how it handles " + "zero, negative, and non-integer inputs before moving to the test file.\n" + "\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "Since the user asked for the validation file first, I'll start by reading " + "shop/validation.py around line 18 before looking at anything else.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check validate_quantity in shop/validation.py to see how it handles " + "zero, negative, and non-integer inputs before moving to the test file.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "call_id": "toolu_0186wos82AWCLwkPWVeVegg8", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_0186wos82AWCLwkPWVeVegg8", + "namespace": None, + "status": "completed", + }, + ], + "usage": { + "input_tokens": 877, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 873, + }, + "output_tokens": 228, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 130, "text_tokens": 98}, + "total_tokens": 1105, + "cost": None, + }, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "Since the user asked for the validation file first, I'll start by reading " + "shop/validation.py around line 18 before looking at anything else.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check validate_quantity in shop/validation.py to see how it handles " + "zero, negative, and non-integer inputs before moving to the test file.\n" + "\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + }, + ], + }, + }, +) CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "anthropic/claude-fable-5-1", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "anthropic/claude-fable-5-1"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", @@ -677,81 +801,95 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, "model": "claude-fable-5-1", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect.", + } + ], }, mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_011Cg343GgaNGYZ8EW5CiKWy", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_011Cg3734Dqb6rpmJnPKii79", + "content": [ + { + "type": "text", + "text": "Reading `shop/validation.py` first to see the rule behind the zero rejection, " + "then I'll read the test file in a separate step since there's nothing to derive " + "from it for the read.", + }, + { + "type": "tool_use", + "id": "toolu_01QKMW1txMKCm8oSZyEwo7wX", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 155, "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", }, - "diagnostics": None, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "anthropic/claude-fable-5-1", - "object": "response", "output": [ { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "Reading `shop/validation.py` first to see the rule behind the zero " + "rejection, then I'll read the test file in a separate step since there's " + "nothing to derive from it for the read.", + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_01QKMW1txMKCm8oSZyEwo7wX", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01QKMW1txMKCm8oSZyEwo7wX", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 155, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 155}, + "total_tokens": 1032, "cost": None, }, - "user": None, - "store": None, "provider_specific_fields": {"citations": None, "thinking_blocks": None}, }, ) diff --git a/tests/integration/translation/responses/bases/azure.py b/tests/integration/translation/responses/bases/azure.py index 1b27e198186..8cc204ecacd 100644 --- a/tests/integration/translation/responses/bases/azure.py +++ b/tests/integration/translation/responses/bases/azure.py @@ -2,6 +2,9 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import RESPONSES_NATIVE_RESPONSE, RESPONSES_REQUEST +from integration.translation.templates.azure import AZURE_RESPONSES_REQUEST, AZURE_RESPONSES_RESPONSE +from pydantic import JsonValue GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -630,146 +633,237 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) +GPT_6_LUNA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 384, "check_offset": 0}, + }, +] + +GPT_6_LUNA_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0CO_lJPFbtrP8DdqMTwl2xlTnCuiBzPQBDrYhcy-xADgVDfRLfYGRE5C1tnchYKxhOh_UwHR" + "p7D0wxyvh3I3hy6iRIr5phgwGqRcJzzeuDaj63bR6wUNAN1iW4EBIamL6eAo3pg-4bzAkACoCbGT7Uwp" + "73E83yafb8_vE9QeJ0lSd5faC7-H5zFe9LDgNywIUNp70Ni1d5v6dAF1HdloQhCkqW0cbDzG0WeeT4jT" + "Yq5yVbiTM_c5PIt3ijbA6OVxbGF5cmtUQb4zOUlR9omxvKRniASAyu9qg201lkwONzoJ7S4rOqROt-1O" + "o8E0OaC1HTxBPjbqSFT4KLH2DkVN76JG5Mkh8UPHySKMKkTmyCPZMwFFjm4-RGgqPQ82KrpfFd9Kwwwy" + "G7ZgIIDG-ckYRW1JvZbew_tsi7zEfOUU7qxeyENyfHbRhe98j61WgLoNuct3ywCkDqTMFe-6YY_AAG8Q" + "FuSc2zfGj6F4UWf1eYiBXm5vzFYgGJLpR1dx8Lthv9gLSKXwyjUDLPev1wkW9QwRGZ0VKUz8r2YC4rT0" + "PwexJzEXoXmWB6unWPL23bKzUsUDvDal1ZYrB5-CnkEFANPxosqZAjeRYtmUj9dbix7uF6b9aeL41fqC" + "RXZU1RKyCp-quWtnAB8E-Uz4gLAMOeLT4-C1kwDP8YMhXS9WLrQ3SYk3YRfcMb2LRPIc2-KkMCKaFKX5" + "_Sov89DzCtGW4VHBqayHzwaKq_qrbnVY5jFOrm-N_A_1Pdrqa4f-oh6TMKlAhFuuUJCKWSMOiydGwtC2" + "jfPgyJVWWST5JyHOlZcmOq4gHDM57QGHlvqDKz7_bN5SChSAmCnVdgervkHw8HKj_VODkEEwA2U-SGLD" + "SoJykHAMhsGjx_BoHxddVUi6p8qJBP9XWa52WpFO8eatHoZBjJ9WW7IF2ztljVqlUeKoEsSVUwkFo2wt" + "i4mhjqjZiaYz6jOz6gYRkJd80UifNe74xisW_IVXTcgT8gXdcf8CVygNjScDDhdzgjUhOhxCjOFNWLc7" + "01FbRToU2qNg3aQQaMdgOPmczA7HeLghBW7IGZmnYCweoFliXXXo08jSXPlXDt1nhzhjm71or81Hk5PM" + "iQj4-DH3aY-a1umCDChEGrMNSZAZTqGBiLwW_W8h0OtFkZ_MHqxRl9fyKYORKQAjlbhakN1Dze2106Uy" + "1WcC4snJwpwxMhd2leREckRD2YpCXDN-WvYwRrDXETUqTJ7-ZLE2gQ8MNK3czzWceHql0pvySKlp9QDo" + "D_NWTywPag5NlfFE74BivE6oc373ZKOPFcwcoRdFS8boNuIkK8V7HDtW02Wet76ucQ-n4cWhWyz-k_64" + "8WY1OBfrTyh4Rup4L2BOCGEjC63YyGMjynnEV0IZ1tfZI19jxxz9ctDjeShUf9gPNEu2CxS8ZQ0Q4NPk" + "8g==" +) + GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "azure/gpt-6-luna", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "azure/gpt-6-luna"}, expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, - expected_provider_request={ - "model": "gpt-6-luna", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - }, + expected_provider_request={**AZURE_RESPONSES_REQUEST, "model": "gpt-6-luna"}, mock_provider_response={ - "id": "resp_0f6bf1d4fb9f1e26006ac4077413a08193b9ef8007e2a67444", - "object": "response", - "created_at": 1791231860, - "status": "completed", - "background": False, - "completed_at": 1791231860, - "content_filters": [ - { - "blocked": False, - "source_type": "prompt", - "content_filter_raw": [], - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, - }, - { - "blocked": False, - "source_type": "completion", - "content_filter_raw": [], - "content_filter_results": { - "protected_material_text": {"detected": False, "filtered": False}, - "protected_material_code": {"detected": False, "filtered": False}, - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, - }, - ], - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0c7fa1cbc998fdea006ad023bcd92c81978c48bfd696bb8c5b", + "created_at": 1792025532, + "completed_at": 1792025535, + "content_filters": GPT_6_LUNA_CONTENT_FILTERS, + "instructions": RESPONSES_REQUEST["instructions"], "model": "gpt-6-luna", - "moderation": None, "output": [ { - "id": "msg_0f6bf1d4fb9f1e26006ac4077472cc8193a8c6b9fd6d98c8be", + "id": "rs_0c7fa1cbc998fdea006ad023bdb5288197913fbdc89efaee0f", + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_LUNA_ENCRYPTED_REASONING, + "summary": [], + }, + { + "id": "msg_0c7fa1cbc998fdea006ad023bdf4b08197a483204dce142776", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "The next file to inspect is `shop/validation.py`, to see how it handles zero and " + "negative quantities." + ), + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_0c7fa1cbc998fdea006ad023be1b6c8197ac0d31ccff25b0d4", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":120}', + "call_id": "call_cODuVBuAwDhnb8M0FPGYyS1f", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 113, + "output_tokens_details": {"reasoning_tokens": 51}, + "total_tokens": 422, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, "id": ANY, - "created_at": 1791231860, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + "created_at": 1792025532, "model": "azure/gpt-6-luna", - "object": "response", "output": [ { - "id": "msg_0f6bf1d4fb9f1e26006ac4077472cc8193a8c6b9fd6d98c8be", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "id": "rs_0c7fa1cbc998fdea006ad023bdb5288197913fbdc89efaee0f", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_LUNA_ENCRYPTED_REASONING, + }, + { + "id": "msg_0c7fa1cbc998fdea006ad023bdf4b08197a483204dce142776", + "content": [ + { + "annotations": [], + "text": ( + "The next file to inspect is `shop/validation.py`, to see how it handles zero and " + "negative quantities." + ), + "type": "output_text", + "logprobs": [], + } + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":120}', + "call_id": "call_cODuVBuAwDhnb8M0FPGYyS1f", + "name": "read_file", + "type": "function_call", + "id": "fc_0c7fa1cbc998fdea006ad023be1b6c8197ac0d31ccff25b0d4", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "status": "completed", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -779,44 +873,15 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 113, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 51, "text_tokens": None}, + "total_tokens": 422, "cost": None, }, - "user": None, "store": True, "background": False, - "completed_at": 1791231860, - "content_filters": [ - { - "blocked": False, - "source_type": "prompt", - "content_filter_raw": [], - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, - }, - { - "blocked": False, - "source_type": "completion", - "content_filter_raw": [], - "content_filter_results": { - "protected_material_text": {"detected": False, "filtered": False}, - "protected_material_code": {"detected": False, "filtered": False}, - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, - }, - ], + "completed_at": 1792025535, + "content_filters": GPT_6_LUNA_CONTENT_FILTERS, "frequency_penalty": 0.0, "max_tool_calls": None, "moderation": None, @@ -825,160 +890,243 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "prompt_cache_retention": "24h", "safety_identifier": None, "service_tier": "default", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, - }, - "web_search": {"num_requests": 0}, - }, + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], "top_logprobs": 0, }, ) +GPT_6_1_SOL_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 519, "check_offset": 0}, + }, +] + +GPT_6_1_SOL_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0CO37PXMD2-KW8EHtGczorILYppFKOeL-058QC8QCnKldc08rJXqRj2qhEP8qBLUBQLaB-Ku" + "bhgoZVZsXiobJCVijCX5SDQcINWwFg_SlMsL2pAHEymUIsT99rxULm11jvG8vfVEsBhBMjhkB-8wCua9" + "q59iXdcK68bVpMhl5BDFsbf1EPIZTY6WFG79yjvXGWqGjsGbCY_EIa8ZsexuLSuZALT1oMI-4KAGDIU9" + "F29ckEi3rV40L1TUZxG_MtrFGBE3d1DKSe0vz5QjJiMaBzZ8eu4kxfjAX5vwfoRx59zNRfjEjgx1s_eF" + "4kCBSZn61pC1ME_yY_omKF4baWkKePFysGvv_qbjE_IH5DSVZN0_WNCNx8ISHtr72p2Niy0vZidXoZVg" + "ByGXlQam3PtAZeRts1wR0VTtb1-xiq8E2hqRXIPmOgBwH35l72tahRBbhtogi_cFxKF-FWYMJOoZQ_dG" + "_gZEEw1vJrepfOZNQBDq62o2h4BI24w5hf85sETMJwaT_3WiLWd0ukdIxK1bLQpwl4_ZEGmkMZqtwCIO" + "iZSRfVTsz8GKcAagqEXa1N0Lz-4ejcy1r6aSaNVdXebzgLySS9wWjlF80xV29HgkUQY1kzJA0VwVmf6m" + "neFdZ9O5yhg3acy_MzALDXbdZ_D4aCO9gtGp06eoO40Cp6fj5CFMrrj4CeC0oV0RrYF3cSb_fSTaKnbH" + "VEbvlMEfuraBBUWTjjYGu7cXQcj7O43V4x4FFM1j_0Lf2i5w5hrHB9BQy5TiySvlmIeDxfXP-wadoxST" + "Gdkek8u6Gu-jivpLKgNDQk7Od-E-V2Hx_xdHx3koV-BfQaNpAzSNIJ9gYpbeySEFA29XXIzpk3QAreHa" + "Z6Ta3yEn717erYIM3QofAOimDyhOUaqFAP4OKD8ljPGa0XAHjJ6iKd3X3oN0ir8072MN5LviK2p3cuwf" + "y7xd_rChJ5KIyXZMa7ZXis-R1a8tUmjVUITS4lRFRmhGgeq5yAwIXSEzUxY-EWPPm2AvC_qVedS9et_N" + "2Bb_QIkmPPOvjbvDce01Owhn9O_n0kRhqlvv72qWqr12Up6eugoazsh05Hvf0BBkPboDEhi23zy0FeN5" + "EaGIJB7D3qe_3awNuq6_2oPVFLT0oQeQegAVyl2Hz5U10mgu_elaX32MV3_54U9o2d2kky17NvAUWrdy" + "GWkerdLWtI3GFjTt5Uk6jIQOhy00PoaqZtv4_XjH8HADN6NcbGikIowF5KslciZKeFZOURaaOtsrwmgF" + "eWlwFaV0JVWrA1Ob25Io7rH1v3pCZFf8tCONGDQGlOmoJ_MLegn4HulHcZ5yGpny36P5aPiU8JnJOnpc" + "xdow0XyjHqS4DFQ9GoQg2IAKWRIk9c8-_PE4nH9OVB_L-3LpZGGWT0gKREmfVmyb2jENTCG6PRFPGgEP" + "9D4GFJTlOo4cSobLe2rKEWl0F-8WxQWCX1CSxq2-Ik3pe08HcDj3hB4JtGVLW-5RGwtvaxfp22m4-aHi" + "oVOPhErqa3wvYgv9i4IE4WJ3yjE3rgEHQ3CuqD9bqskSh2WHqJwfX6wb-2etzdFiZPix74VGlaMZR8Hr" + "Xh8IEHUVj_uL" +) + GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "azure/gpt-6.1-sol", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "azure/gpt-6.1-sol"}, expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, - expected_provider_request={ - "model": "gpt-6.1-sol", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - }, + expected_provider_request={**AZURE_RESPONSES_REQUEST, "model": "gpt-6.1-sol"}, mock_provider_response={ - "id": "resp_0da7f0a44a7da608006ac40775cfbc8197991a6f98f5111832", - "object": "response", - "created_at": 1791231861, - "status": "completed", - "background": False, - "completed_at": 1791231863, - "content_filters": [ - { - "blocked": False, - "source_type": "prompt", - "content_filter_raw": [], - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, - }, - { - "blocked": False, - "source_type": "completion", - "content_filter_raw": [], - "content_filter_results": { - "protected_material_text": {"detected": False, "filtered": False}, - "protected_material_code": {"detected": False, "filtered": False}, - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, - }, - ], - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, - "model": "gpt-6.1-sol", - "moderation": None, + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0895a553a79167f3006ad023b495cc81978a0ed0ada7341347", + "created_at": 1792025524, + "completed_at": 1792025527, + "content_filters": GPT_6_1_SOL_CONTENT_FILTERS, + "instructions": RESPONSES_REQUEST["instructions"], "output": [ { - "id": "msg_0da7f0a44a7da608006ac4077719f48197830b80e5759a5d75", + "id": "rs_0895a553a79167f3006ad023b5d6348197ac3ab4cda17f113c", + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_1_SOL_ENCRYPTED_REASONING, + "summary": [], + }, + { + "id": "msg_0895a553a79167f3006ad023b6ca3c81978c47c3a9216638f6", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_0895a553a79167f3006ad023b71d648197afe5314dfd20909f", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_h9vdLDpm7EuHrkHXzdKY0Yb2", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 140, + "output_tokens_details": {"reasoning_tokens": 75}, + "total_tokens": 449, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, "id": ANY, - "created_at": 1791231861, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + "created_at": 1792025524, "model": "azure/gpt-6.1-sol", - "object": "response", "output": [ { - "id": "msg_0da7f0a44a7da608006ac4077719f48197830b80e5759a5d75", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "id": "rs_0895a553a79167f3006ad023b5d6348197ac3ab4cda17f113c", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_1_SOL_ENCRYPTED_REASONING, + }, + { + "id": "msg_0895a553a79167f3006ad023b6ca3c81978c47c3a9216638f6", + "content": [ + { + "annotations": [], + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + "type": "output_text", + "logprobs": [], + } + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_h9vdLDpm7EuHrkHXzdKY0Yb2", + "name": "read_file", + "type": "function_call", + "id": "fc_0895a553a79167f3006ad023b71d648197afe5314dfd20909f", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "status": "completed", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -988,44 +1136,15 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 140, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 75, "text_tokens": None}, + "total_tokens": 449, "cost": None, }, - "user": None, "store": True, "background": False, - "completed_at": 1791231863, - "content_filters": [ - { - "blocked": False, - "source_type": "prompt", - "content_filter_raw": [], - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, - }, - { - "blocked": False, - "source_type": "completion", - "content_filter_raw": [], - "content_filter_results": { - "protected_material_text": {"detected": False, "filtered": False}, - "protected_material_code": {"detected": False, "filtered": False}, - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, - }, - ], + "completed_at": 1792025527, + "content_filters": GPT_6_1_SOL_CONTENT_FILTERS, "frequency_penalty": 0.0, "max_tool_calls": None, "moderation": None, @@ -1034,160 +1153,244 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "prompt_cache_retention": "24h", "safety_identifier": None, "service_tier": "default", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, - }, - "web_search": {"num_requests": 0}, - }, + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], "top_logprobs": 0, }, ) +GPT_6_ASTRA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 567, "check_offset": 0}, + }, +] + +GPT_6_ASTRA_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0CPLsQ7EOhGsCUWnH7tYlsJHraBIt-auus4qIwbzqaCzNqSCmiefHbKwVcOnreZc_AwoO-HK" + "M_0Ataa_BhrACZn7EkzTtUPaypdS0R9INoJ6hsR5vQ8YWibtIffS3BR5XaxsHpCcqFiNrIXvd0QvefYv" + "vJ7nyPfJvPxCAw0UoWU-XTXeexjM8twO6AvDlTqzP5yCG0hwmYco7zeiwWGxoleRMRoEiYIiiHwWxPXZ" + "nr9DuZeyqKFgByJsZ19pUAQSlyCKQkKEvMw8KMyPn8iYqxXaQ1vgUBIKopAk9UBkdwDZ9Zd03ZjLodo-" + "UQx6H0gwQQHVBTej7HJ-52cT0pOJ3vsPjKEx6m_RSw6-s648hLZNWzDYqgfVI-UGID2kw_qtcYQtVhDG" + "GikbbkwpgJPYxnEmVw3ywlrXxGg9Crlgy0K8S0F1jNkeTuPAMkMRkZOCjiii2DyYrwYAAQpb6ARZEcUv" + "mMKlpPyhW70JZ3qBHLsSxCGAMaTNsiKPxLHai_nCgoGwrbNTspoCNy55RSDqu2jlhaGfUiF6u3j1Tw_k" + "zkPY45URBKIaQH055M1D48QxOmci903Rnai715k4RGjHyG30kqCHeZ9F-C7CnRnEENZPi69-U6pIvNF3" + "s9PkIgXxnOQgBTVO8s9Q1cLoooM5N4s_KFIrESrJCivCebrrOGUaYUFpZ02FS_UGu_CS6Oms8HroLSsL" + "bKoTD3AO_n5xM5lFu3eTtFCXWGK0_IaR_dMQ_s-bdyZ8KtcrHlG_UKR9TaluS17kGtokuTkaWlQMCvf4" + "zIUY3mcZU44dz7EuMXupipNLkxrAKZFL6ypcaqjGgSN2eTQFJCO6oCbkI34vbg7-2524ttNkuTXNRCTt" + "qrbcNOXoDS8asvpa7JbYsKmtLD5bCZGyR_tOgbjSBVmfnT6eQsh2Y6dIf0UJrvCmtOt316p1mydz7-Ek" + "DEFuAJfTTk7H2zhFncIa2rej33J52FkOIHarQYr9gFG95gTXtqswWuOQ10_C9EG4pd3xJiG9BfC7sefP" + "XFORAIQ6DJ2rwJWTLixquGrdEN9oe1agij-jRGi5kXfcu12s8bw8d77wW6T9hDDTepEJ8qpNC-DXCvZw" + "Kt8CxMQvPoTH8nJx9zeeu8mbahkTBLQ_DS18bbFvExbyzDWASZ6oOZD0Ni8mxIxlh18ZzhzHOq82XvH5" + "YdYkuivNezw-SvImaCXKCDYvV8kPxNnT6D7SCYwJWC4FEnc9ZHw1JthXMVt5NZJLGs4kQS2p2HsfMLMk" + "rFgIY7Ra4WXvLduVcxuqa2lV3y3Rp5b5Vx8pkIRtHdI4bNfcV5nVsXskjIlRdLZ_nAzEZKbPFxuVnS3r" + "KXZi8LNgl138jVxid4ETBk5-YmnogzRYZZ3wb-ULgDP7Y3VlXbkqniAbDY1BsZK9I7V5IgEgbEl1J0Ck" + "fjH9Q5m1aeLBA1vqLtEMtwVEvM78mNctefN9LOCIXGzWKvVPs72ez-DQ71HZ_nVexKpvGKwoPTVCn92M" + "xs5l9ZT5YSXu0ifdAEKo3334mU_DqyvFqpUeVkaRnSg6rHTtnkraG6ieExe9cGguMdUHAb47VEVCgMHg" + "9ItzhlhOLuX82fHONdmUeWP04AQrZTI3U5OaN8C15GdirZAc87uQzEJ0Eh54uOLDcS1wqsSk6Hu6" +) + GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "azure/gpt-6-astra", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "azure/gpt-6-astra"}, expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, - expected_provider_request={ - "model": "gpt-6-astra", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - }, + expected_provider_request={**AZURE_RESPONSES_REQUEST, "model": "gpt-6-astra"}, mock_provider_response={ - "id": "resp_0619fdc05c5504ee006ad01712e1988190bcc283ea6c2aa02b", - "object": "response", - "created_at": 1792022290, - "status": "completed", - "background": False, - "completed_at": 1792022292, - "content_filters": [ - { - "blocked": False, - "source_type": "prompt", - "content_filter_raw": [], - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, - }, - { - "blocked": False, - "source_type": "completion", - "content_filter_raw": [], - "content_filter_results": { - "protected_material_text": {"detected": False, "filtered": False}, - "protected_material_code": {"detected": False, "filtered": False}, - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, - }, - ], - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **AZURE_RESPONSES_RESPONSE, + "id": "resp_09fa191d4c7fb192006ad023c7952881958572a6686bbffea3", + "created_at": 1792025543, + "completed_at": 1792025547, + "content_filters": GPT_6_ASTRA_CONTENT_FILTERS, + "instructions": RESPONSES_REQUEST["instructions"], "model": "gpt-6-astra", - "moderation": None, "output": [ { - "id": "msg_0619fdc05c5504ee006ad01713e44881908c0b1c9723b75e0e", + "id": "rs_09fa191d4c7fb192006ad023c898e88195a30e3b694f965fb6", + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_ASTRA_ENCRYPTED_REASONING, + "summary": [], + }, + { + "id": "msg_09fa191d4c7fb192006ad023ca32f0819596d7f46dd3c29f03", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_09fa191d4c7fb192006ad023ca9800819586b8ea16baef56cb", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_PuG8zuJ8TpdNdpQFRquUj6ZL", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 152, + "output_tokens_details": {"reasoning_tokens": 87}, + "total_tokens": 461, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, "id": ANY, - "created_at": 1792022290, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + "created_at": 1792025543, "model": "azure/gpt-6-astra", - "object": "response", "output": [ { - "id": "msg_0619fdc05c5504ee006ad01713e44881908c0b1c9723b75e0e", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "id": "rs_09fa191d4c7fb192006ad023c898e88195a30e3b694f965fb6", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": GPT_6_ASTRA_ENCRYPTED_REASONING, + }, + { + "id": "msg_09fa191d4c7fb192006ad023ca32f0819596d7f46dd3c29f03", + "content": [ + { + "annotations": [], + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + "type": "output_text", + "logprobs": [], + } + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_PuG8zuJ8TpdNdpQFRquUj6ZL", + "name": "read_file", + "type": "function_call", + "id": "fc_09fa191d4c7fb192006ad023ca9800819586b8ea16baef56cb", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "status": "completed", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -1197,44 +1400,15 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 152, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 87, "text_tokens": None}, + "total_tokens": 461, "cost": None, }, - "user": None, "store": True, "background": False, - "completed_at": 1792022292, - "content_filters": [ - { - "blocked": False, - "source_type": "prompt", - "content_filter_raw": [], - "content_filter_results": { - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - "jailbreak": {"detected": False, "filtered": False}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, - }, - { - "blocked": False, - "source_type": "completion", - "content_filter_raw": [], - "content_filter_results": { - "protected_material_text": {"detected": False, "filtered": False}, - "protected_material_code": {"detected": False, "filtered": False}, - "hate": {"filtered": False, "severity": "safe"}, - "sexual": {"filtered": False, "severity": "safe"}, - "violence": {"filtered": False, "severity": "safe"}, - "self_harm": {"filtered": False, "severity": "safe"}, - }, - "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, - }, - ], + "completed_at": 1792025547, + "content_filters": GPT_6_ASTRA_CONTENT_FILTERS, "frequency_penalty": 0.0, "max_tool_calls": None, "moderation": None, @@ -1243,16 +1417,266 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "prompt_cache_retention": "24h", "safety_identifier": None, "service_tier": "default", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, - }, - "web_search": {"num_requests": 0}, - }, + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], + "top_logprobs": 0, + }, +) + +GPT_5_6_TERRA_CONTENT_FILTERS: Final[JsonValue] = [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 297, "check_offset": 0}, + }, +] + +GPT_5_6_TERRA_ENCRYPTED_REASONING: Final = ( + "gAAAAABq0CPSc68PgJKpbMkIorqdkFkGusPg2cNuR1B1t7kGmuehEWf0bpLWzdkOUkQgxYbzawyuBLR5" + "sb-Wflsld_-Qj5WlHfYZmRJU7n21c2JKaWMtvoEf2WwktXCuHJCwg9hwBNUhIGuLjIJFqx4gsHMcoNSZ" + "reXGm51nnzkzX48zanXTWOgfELEUNPazQlPkhMWsO0ZvjomVM-OaWVWCzvD0MIoAe42ZPAeA_448XG1k" + "noYd5r4kxIn2ggODmjP2U-ijPUkLWt1ulClD2k8B_tGPUIAfMfN_jsx1KLLbLw0cppoedYXYl_PNWJn_" + "N3L0Us5Vtj0QLwO1z4MfNus_oTJ4PLa3K43al71izDw9gJlWZmT_Tme34cOuTb5_3O3rPSMSMVvNoXjG" + "MjNzlL0lNQwu5O-scO8U-6nwfSZt8gSkQ5h5GflqHLgFQkOFWFu9tAcifvaTjznOFlinK43vpGIf796d" + "ZFq_MSIBgGdnNGEUsDsFvMUXomn4Zlw2_v9icKiaFr505BJehAgj4f7WRI4vnIJCMI5WByUupPB9Rawi" + "rB_b5OhDLfrI9jEDE8DtkVKCGf1nwrI99Bllt9-EtyOTSWspKOzDinbIkS7bhOXy0oZh_8_6vFJDdzzV" + "C2yRRfHPVUGndylD--H5P-lkgDN_qwWTDIOhhTd-9TnewshMmrah0EfNimFLaxYWQ7vqBA1C5_Dh9gAd" + "5SZ3Pb0C4iOjsHepbUl4l-9DtrGIGoJ4pz0XukOZRa-SXLxHOAI2eil9Rnj08MR37yYy_lrIjq75F_0R" + "d-11MzV0_ej6UDABd-h9Myzn_0WR-qpBfnCyH-z9w_SB8X7ErzXOImlzSvIo4B-Oy_xSLVwSjNRW3lbW" + "Z8uoNHPXFj3miv5w4lb2TZTd_6ltOeH-I746jdKa0jxgZOY2oHEKH2eFEqYjQvNd06TGMvJYByM1urkI" + "h1GnTRH9VVoHa7QbyLAxL9hdV0pGc5Rrh9eIvdz830irEwVO6nS8t6tXQ2DgKy555nmlVQ6_inLzvw1t" + "uU0di7-JZK28JCgo7zMw_UwIDok7oeLXFAbUtlVNvyp2mJ5kz9cm-xK7veFgiKNrRi3NqospaOfEt3NF" + "_NV62O8WKwudIXpg2aTEnxI66GNphwllNQSoZjYrsvMIsTk6GNdqVE7OXpKTRhmVEWh2vb8u4GSMZ7sa" + "Kqf6ju6RbjQF1FyazaFCuBGIVANnTPLPcvt7HIv6HzyUb75S1sjoFimhyRByRn906fCM218=" +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "azure/gpt-5.6-terra"}, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={**AZURE_RESPONSES_REQUEST, "model": "gpt-5.6-terra"}, + mock_provider_response={ + **AZURE_RESPONSES_RESPONSE, + "id": "resp_0e1796d6e4004e2f006ad023d108548193a6493b2aac71acd0", + "created_at": 1792025553, + "completed_at": 1792025554, + "content_filters": GPT_5_6_TERRA_CONTENT_FILTERS, + "instructions": RESPONSES_REQUEST["instructions"], + "model": "gpt-5.6-terra", + "output": [ + { + "id": "rs_0e1796d6e4004e2f006ad023d1f5ac81939dc5121afcbd9967", + "type": "reasoning", + "content": [], + "encrypted_content": GPT_5_6_TERRA_ENCRYPTED_REASONING, + "summary": [], + }, + { + "id": "msg_0e1796d6e4004e2f006ad023d2347c8193a86acc05aee89b6f", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "The next file is `shop/validation.py`, which contains the validator and will " + "show how zero, negative, and non-integer quantities are currently handled." + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0e1796d6e4004e2f006ad023d26b5081939200dbb9bfa0382d", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_IuSjfDiujWaxiYl7MptZoMXp", + "name": "read_file", + }, + ], + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 95, + "output_tokens_details": {"reasoning_tokens": 24}, + "total_tokens": 404, + }, + }, + expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, + "id": ANY, + "created_at": 1792025553, + "model": "azure/gpt-5.6-terra", + "output": [ + { + "id": "rs_0e1796d6e4004e2f006ad023d1f5ac81939dc5121afcbd9967", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": GPT_5_6_TERRA_ENCRYPTED_REASONING, + }, + { + "id": "msg_0e1796d6e4004e2f006ad023d2347c8193a86acc05aee89b6f", + "content": [ + { + "annotations": [], + "text": ( + "The next file is `shop/validation.py`, which contains the validator and will " + "show how zero, negative, and non-integer quantities are currently handled." + ), + "type": "output_text", + "logprobs": [], + } + ], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_IuSjfDiujWaxiYl7MptZoMXp", + "name": "read_file", + "type": "function_call", + "id": "fc_0e1796d6e4004e2f006ad023d26b5081939200dbb9bfa0382d", + "namespace": None, + "status": "completed", + }, + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], + "top_p": 0.98, + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 309, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 95, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 24, "text_tokens": None}, + "total_tokens": 404, + "cost": None, + }, + "store": True, + "background": False, + "completed_at": 1792025554, + "content_filters": GPT_5_6_TERRA_CONTENT_FILTERS, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": AZURE_RESPONSES_RESPONSE["tool_usage"], "top_logprobs": 0, }, ) diff --git a/tests/integration/translation/responses/bases/azure_ai.py b/tests/integration/translation/responses/bases/azure_ai.py index 102c45777ff..921fed14b45 100644 --- a/tests/integration/translation/responses/bases/azure_ai.py +++ b/tests/integration/translation/responses/bases/azure_ai.py @@ -1,7 +1,10 @@ +import json from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import RESPONSES_REQUEST, RESPONSES_RESPONSE +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -99,104 +102,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "azure_ai/claude-haiku-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/anthropic/v1/messages", - expected_provider_headers={ - "x-api-key": "synthetic-azure-ai-key", - "api-key": "synthetic-azure-ai-key", - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - expected_provider_request={ - "model": "claude-haiku-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_011CfoboMutM3uKRT9qW3upD", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "azure_ai/claude-haiku-5-5", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 23, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 23, - "video_tokens": None, - "cache_write_tokens": 0, - }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", @@ -681,17 +586,23 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( }, ) +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQShAYKEAgSGAI4AUIIdGhpbmtpbmcSDEZBSweh3n+zsTFBbxoMzZxq5YKILAjNCjcbIjA/E3kpf/AnHsoj/+k5/+kv+0Zyyl0oHryOrbao" + "+0ht+2v/fbBUPGLQ38xdnNmiFE4qoQXePJCj6Q5is8kKxhEYNop4kKbvzRimV2ER8yql54QSaKONKuCgt0W52r1s0b+7iLw+P2Le5sju8XVy" + "9/c9h06rqUQu2ALeRsVDdTecLqPBWdfLtuePuMrRmLBDNERphdLekxUr9ABAFts/hhybR/HLiDSMkj08Axp+45skaChj7TPaGAQVg/dw7cWi" + "QSA4atcKZk+CfkUu955LX6qaDRoNUOAC4i15y4VbfForUdVk7LBgvI4ynDZ0L4lAtwSIcfNj7sCBY6zyPAD/POQkuTo2jm2UtbMVx4bC6+Ci" + "B6hAtvmA/MY6iIVUTGtordaFtKFcc8hTgr4uya2jeOp1pGN9dmABK2U4wIbyeCiYhJYskqQiVH3Va82hIHkg9oJvc4/4TMLT6nhmuaBsR9Eo" + "8VgFlwmkw9gXF+8FMD/rPdwLjr7aHvyNDe2gW9YxEaiHk5g0ldtyRPj5hLZpMX4CJ1Uqjf2k5LJ5kXljJihoT5/Je516iVPoA0V7alAr1dOY" + "Dxfajmd2o16yOrmJox/BwUGGqh4kPTgaavIKMWNwjaSeGTmd+pccQt4aKEzIk+X/Cst1UOS0vVVGdu/Z0q7MyrRMbfkne8B0i1lPRVhNtNwb" + "32fig40YgYmLIagkHA+jXhBxvLUaB5Xj/TtEMXe6ZVdq7A3fJ72baBTcDQ5uXW4yd0G12QrFEaA2TGt8PNZsA2/G1ZW/iU6oZZIKx5o7Sgeg" + "KNQk/tdXeS3cru7/bIlO6jMaGhYjDn3g42XKj16i6ri5WUVrrFeXq8Oq2Oh3IGIw+tL9RVf/lI/vp487EJcnMPiwWYeuIRwyyN8x/XiudCeE" + "S8mAFwadQyCUHYyeOr/a2tRXwx0o0/lpXb4htqEZHdkOJmXtlR4Whda4NrzO7utcGAE=" +) -CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "azure_ai/claude-fable-5-1", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "azure_ai/claude-haiku-5-5"}, expected_provider_endpoint="/anthropic/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-azure-ai-key", @@ -700,81 +611,252 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={ - "model": "claude-fable-5-1", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-haiku-5-5", + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider zero, " + "negative and non-integer quantities before choosing the next file to inspect.", + }, + ], }, mock_provider_response={ - "model": "claude-fable-5-1", - "id": "msg_011Cg344Nycw3eBjJcjxMMpq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-haiku-5-5", + "id": "msg_011Cg37NSjwYJSDCvonTBaww", + "content": [ + { + "type": "thinking", + "thinking": "I need to look at shop/validation.py to understand its contents, so I'll read " + "through the first chunk of the file.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py`, starting with the `validate_quantity` function found " + "in the search.", + }, + { + "type": "tool_use", + "id": "toolu_01NiAAsMCJRXggv46Zro96cN", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 205, + "output_tokens_details": {"thinking_tokens": 73}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **RESPONSES_RESPONSE, + "model": "azure_ai/claude-haiku-5-5", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I need to look at shop/validation.py to understand its contents, so I'll read " + "through the first chunk of the file.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "I need to look at shop/validation.py to understand its contents, so I'll read " + "through the first chunk of the file.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I'll read `shop/validation.py`, starting with the `validate_quantity` function found " + "in the search.", + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_01NiAAsMCJRXggv46Zro96cN", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01NiAAsMCJRXggv46Zro96cN", + "namespace": None, + "status": "completed", + }, + ], + "usage": { + "input_tokens": 877, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 873, + }, + "output_tokens": 205, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 73, "text_tokens": 132}, + "total_tokens": 1082, + "cost": None, + }, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I need to look at shop/validation.py to understand its contents, so I'll read " + "through the first chunk of the file.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + }, + }, +) + +CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "azure_ai/claude-fable-5-1"}, + expected_provider_endpoint="/anthropic/v1/messages", + expected_provider_headers={ + "x-api-key": "synthetic-azure-ai-key", + "api-key": "synthetic-azure-ai-key", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, + "model": "claude-fable-5-1", + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider zero, " + "negative and non-integer quantities before choosing the next file to inspect.", + }, + ], + }, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-fable-5-1", + "id": "msg_011Cg37P2hrNhVAyq9ahS7H9", + "content": [ + { + "type": "text", + "text": "I'll read the validation function; reading the test file at the same time is " + "independent of it, so I'll fetch both now.", + }, + { + "type": "tool_use", + "id": "toolu_01XZZ8P93YHKzXM5XGRZN7hs", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_01MpgANqoXoBcnhnsz81Q4Q7", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 232, "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "not_available", }, - "diagnostics": None, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "azure_ai/claude-fable-5-1", - "object": "response", "output": [ { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "I'll read the validation function; reading the test file at the same time is " + "independent of it, so I'll fetch both now.", + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_01XZZ8P93YHKzXM5XGRZN7hs", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01XZZ8P93YHKzXM5XGRZN7hs", + "namespace": None, + "status": "completed", + }, + { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}', + "call_id": "toolu_01MpgANqoXoBcnhnsz81Q4Q7", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01MpgANqoXoBcnhnsz81Q4Q7", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 232, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 232}, + "total_tokens": 1109, "cost": None, }, - "user": None, - "store": None, "provider_specific_fields": {"citations": None, "thinking_blocks": None}, }, ) diff --git a/tests/integration/translation/responses/bases/bedrock_converse.py b/tests/integration/translation/responses/bases/bedrock_converse.py index 621609a946e..b0a2c881878 100644 --- a/tests/integration/translation/responses/bases/bedrock_converse.py +++ b/tests/integration/translation/responses/bases/bedrock_converse.py @@ -1,7 +1,15 @@ +import json +from collections.abc import Mapping from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import RESPONSES_REQUEST, RESPONSES_RESPONSE +from integration.translation.templates.bedrock_converse import ( + BEDROCK_CONVERSE_REQUEST, + BEDROCK_CONVERSE_RESPONSE, +) +from pydantic import JsonValue CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -86,90 +94,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/converse", - expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], - }, - mock_provider_response={ - "metrics": {"latencyMs": 954}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", - "usage": { - "cacheReadInputTokenCount": 0, - "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, - "serverToolUsage": {}, - "totalTokens": 29, - }, - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 23, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 23, - "video_tokens": None, - "cache_write_tokens": 0, - }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, - "cost": None, - }, - "user": None, - "store": None, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", @@ -419,252 +343,669 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( +BEDROCK_CONVERSE_RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in RESPONSES_RESPONSE.items() if key not in ("provider_specific_fields",) +} + +BEDROCK_CONVERSE_RESPONSES_SYSTEM: Final[JsonValue] = [ + { + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "responses client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", + }, +] + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQS3AYKEggSEAEYAjgBQgh0aGlua2luZxIMvtg77qfREr4ZWBn7GgzxzrtNwc5N8DqpBHgiMMJUcH5Vw6hj0kYyAwbI34q5FDQHs+VeRDCU" + "N9EDE5JENmjhQSbLvD87AP+BtUfZPSr3BegcgjSctkyUvfkBNE3E+FvfkZAAwgvBMZ904om1Ua3usYegKOyyc4dHlwRSvDFvJFDSeIrtTek2" + "ugzdsjeSeJLr8FoiYxqVIDSfe40IraTHZs5omadfA/F2I9Yj6nGHUCkGQK/TKZWA4G+3+H4FiPK84gjyFL4686p/8ljiWsz11SO862UHRR7J" + "QAl9e6tprj4qpJRawR58Ky+JyOAUzUD/DpZyFYjYFwsgAiv5IY7kCfW/osGKrwnEr9dYX8JQu6h5kZBxFBxETdASJQBHrIWC/zrYADuzIO7Q" + "YzRwnQqc0v/hCnuf01rBX+q41iRVHlJIDsEmdkxiNB4tGE/Azo+RUbTd4fBBGH2t9iY4zi+5ykBZ38BzjTIQAjBc5Ip3LSoNHqNzFAj65wBj" + "nI1L21+jANpMxjZ1DvXCkMc2ZMh/hkPNGqnEKPaIZjYTp6ZX4MJ1G7orq1VqFUYKJ39whqkQR2i64iaBRJ7f+n4eHYNR0ZAGF6E0/pJzpB27" + "2lsWrz2Ky6OsK0UU5HsGgFmkOLI1IGEvXqeE4gmRz8AMseNY1RobNPmNmv4nj7DjLrg9wnkDLlQP0Vg8N6LXqGPlIZeICtcNEM/7KhiqIwzf" + "KZnPc5erpBrCS3qcM0jzP/vGwOdpuZWbiDfn8iwJatVEYgwzsZOi1kIJDGHSCLyznM8CiGGpTBY5BIh8yj2k6D8JIOdKOLfVVJtrCE+LowpB" + "BTGLnC9mEQkJ4ZD/wS4rHQOeWiN2tvbm6casvCvzDDRPISoLI0mQa7WXr+ovbScBIgmZhoJFAwaU5b0ii5hoizTDx6Uc4Ak5zr/DETMNvVde" + "dTvD0MWzZ0Olo1W3bUoHfu+I7yiPGE9jmp3Yugj293UtIUm5GDJpuXwDeDTrcu+dYrFCWm68qn7dRAk2/47a9BqknHboVsD+Z0Zxi3DItUBa" + "/iFxdh7kgJSVZ+oHy2EcEu3Tlg1vEDDPXoivGCzV/sSQ1L2kimTI2jtYhHkWZU7kn7VUmCXdsRgB" +) + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-haiku-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_RESPONSES_SYSTEM, }, mock_provider_response={ - "metrics": {"latencyMs": 892}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 116}}}, + "metrics": {"latencyMs": 3114}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + "text": "I need to check shop/validation.py to answer this, so I'll read the first " + "chunk of lines to see its contents before responding.\n\n", + }, + }, + }, + { + "text": "I'll read `shop/validation.py` first, since that's where `validate_quantity` is " + "defined and where the zero-quantity rejection should come from.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "name": "read_file", + "toolUseId": "tooluse_l8Pc3cjAkezdGOeQoxs9XG", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 853, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 853, + "cacheWriteInputTokens": 853, + "inputTokens": 4, + "outputTokens": 262, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1119, }, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", - "object": "response", + **BEDROCK_CONVERSE_RESPONSES_RESPONSE, + "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I need to check shop/validation.py to answer this, so I'll read the first chunk of " + "lines to see its contents before responding.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "I need to check shop/validation.py to answer this, so I'll read the first " + "chunk of lines to see its contents before responding.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "I'll read `shop/validation.py` first, since that's where `validate_quantity` is " + "defined and where the zero-quantity rejection should come from.", + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "tooluse_l8Pc3cjAkezdGOeQoxs9XG", + "name": "read_file", + "type": "function_call", + "id": "fc_tooluse_l8Pc3cjAkezdGOeQoxs9XG", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 857, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 853, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 262, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 116, "text_tokens": 146}, + "total_tokens": 1119, "cost": None, }, - "user": None, - "store": None, }, ) +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQS6gkKEggSEAEYAjgBQgh0aGlua2luZxIM5Rk71l/Z9rkMtA6ZGgwuf1QIpmmol0EDZgwiMAXhxIVSflqszyyicoACM0MZMWNy03vjepAm" + "b4tmo7kCof9sNzr0fhQHwTSdDGzUlyqFCdgG/z4e8K7FKJQXQvtfpj0ubUZ0e3Cipo/DjMOZdQlMuAy+mywzO73iQG/afjuWcEMc0NKMgPki" + "0EFN23Fy6adXfSQXvRej3PxBT16bNSth1ytGBnoby/VgtMa1JVW3E9h9kCkbEnA8+AT1XSgqQs1pigN5LbTPDN13nYutXxd3BoP/cnPsR5vN" + "znslFbfjd19eup0W0iKTk9xpP1HEvcuYnYlIIVDHYGDP7rzUXtWIqI9ZG1KdzVHFBa9qhEcmKMeWks4iXcKVAiNRcdip3kdowlGZWyG4MEmJ" + "94BWm9JJbn6f+DpxsDk6bFyqq1RJW5l+aApuKX/yN915Ccd6gARvz+nuxpjuDZOln/DzHiBqlnDBPRgn82q+csvQ3iXAxIeNEcTscEu+zsIU" + "zwtZciFKXs1TlMIiBAtoQ3NE7HM9vJNkWWiQRHtE5n4jhtN2Yqs5hnvg05vTGvXnfqs5sGjbP2yj18TOYmu2HlZd6NFADuLTulA/zovgGm5t" + "gNzJE26Blh/4QoiT3TMUMjvOJtRhPz2VmdkKbbiBn1R3Nf1CTStND00zdOQ6NQululcU5Y74fbnzADHl1dOAw7AkoHlp953vkEOJvK1N9kPw" + "ZOwBz5Nc0XRRoZ4zXSM50wg6OdTWqS8vT2nlJslvKT9AghPV3LHyY5HBUpZPR0Up2OZQKZr/G5/BAKpM5Jr5P0tIAf9gHLBiHNKcpk+Upkzv" + "HfT8QV3dl0x80yaUyQrU1Glv8K56pI3OUe/NkmJw7CMhVSUfPt4U609XzzKNEEwFKUjoUtJt/ZOsBxESqGfs3hSayF6i9kFzIzWdHKE6RnAj" + "7JtjVQlff0aIpoAH89oq5ZhFaA8xkqbxu8eDY0L4HtNRVTEyNiGrnJBlp0O7eEoL/4ZRW+j6HcK9bB7O/8diJ/JXk3MEpyYt8MLBHbhVbzzX" + "PJuIZM5NObOchFbR53EjBHMvyRVvF4SnZaZAW3DxOeA49UpuiBDbDPiPe0XOmtM3BwK+vRnmeI61nWfn4mAOVQeNZLjf+spO+3bDxXtHt1bz" + "yK1Z4pLO3dkBEhZKEk6Ha6fRPSPBi2/xcK80R1WyWbVhGvgYK0hP9Uy36ZK6WvTMiEVL9b5/+t+yNNJsflstuMYLNuASU0PE7ne35zYao/gc" + "qKzM8t9gjJpqGUtPoR9ndaBlilgHOkgXV+h29jA4OIPtJnFfI49sztRrA0thymK+lm8LY0F1nnhsPkxc1dohz8EykdV18vO8y2cyeZHNwVrp" + "OEW3nWGz+/GFyRCKsfexlGk6npiFvk5ZJQFZY/Kdh4VuTDKEhmm9QhZTwEYFrh3B5OlO8UjaiIliJ+4Xx642I2MvRbkeqDbJGxSCqxNIS1yJ" + "Yq9UVZzkFXhPvy9/9XFfXH3tedoDceTGG8YD4IummDngMs204wz7VHve814/eTD3qS70oX5O36aY/Aswnz9lS4PhYrAtITp0YmNyBingiqpi" + "Csxq8ePR2IQXawltEYg3cS+weEVJBUaxusjRf/3SdwhyoDuXqaqtrr/r4L7UMf5HGAE=" +) + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQS9wUKEwgSEAEYAjgBQgluYXJyYXRpb24SDNP7b6gtJ4xvIEG9ohoMqC7J3vxdFt/qo/sOIjC6trR5FvvYWW/MCffxtMIugtRws7utg3y+" + "18ZIygEbDw4yGfS2DkcGZ9q6obx1Zs4qkQV27sYDMo7sDReqPo0XpOQHMt7NOsJmrJQ9IdAfTOXPqKPwnC6LU3dnS6lvSWw5gDrdXgXd9iHu" + "Yb1+pVCrkpBnYwrb5N7tOqMQvRI60qxGBOarSLTpvhysv5G3ZIXfmliYZbmITa3I8tlvKoqIIAGXc7zqgI7VfMU8yuK/dtDbCO6/Q+/qPnIL" + "lrY+F4NIaByk9WcjJMXGarMfuohw2acciumMpDu1rqxUpLaPMjhaTFv17lu07uOOu3cuiySN06Fzwp7TbWiKS2pi1aORKXYDowrF59gyTppc" + "G6Y3M5983lXB2oLtz7Gnyr8uBbcjayWDqZCMitoa8m5twh+VEOnHdzay+a4glnSiPr5A1D3nH1ENocWj6+eKS+LXkdvOUse5XkDaOUPAfAm1" + "4f3tn9G1+0S6Zo6Lmwjw/+79OfUPFEHgkZ6Ih6xhMR0yM/3FIAjSXMm+t7VNH23Ei7EkCWdv4OrSPA5PweCX8lnMwQ5avMuna6ZeiAV3A/tu" + "SGy2tYkbjpUJ3JPGfuRmwfCV499fkJJY6oZthK1ot0746frbEgHAnYFyts064x+/yp9D0KariKPuCpfyFqDYn4EhoA+ROvBTxkrQu1KveTAC" + "55/v1tOBlLAZsVzXT7XqbCItMt+AXQSA3jaOmtrU425oOyhc9AUKSxvSR+jTBCcGmJCWKrnEWUjqTW5AbG5byposBH3tYor6L769fli2cTRy" + "IerlH8AHj2CgjuXNtGbxP2Acjf6TjGyY2WeIhT/dQXWKQNzfsvdVtfLRyEESr7QKyiBC/vOFeTTMfQoFB7+Q6mTh6mnYmw52ytfociSKPi3x" + "yAYXpJQGxmCINv++wCIocN3xp703kU+Yt+9I4wKL2+JuN7IYAQ==" +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_RESPONSES_SYSTEM, + }, + mock_provider_response={ + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 235}}}, + "metrics": {"latencyMs": 5698}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + "text": "The user asked to read validation first, though both files could " + "technically be read independently. I'll follow their sequencing and read " + "the validation file first.\n\n", + }, + }, + }, + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + "text": "Now checking shop/validation.py to see how validate_quantity handles zero, " + "negative, and non-integer inputs, starting with imports and helper " + "functions near the top.\n\n", + }, + }, + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "name": "read_file", + "toolUseId": "tooluse_HZpviLzbdTE4T0GMawHsmi", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, + "usage": { + "cacheDetails": [{"inputTokens": 853, "ttl": "5m"}], + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 853, + "cacheWriteInputTokens": 853, + "inputTokens": 4, + "outputTokens": 404, + "serverToolUsage": {}, + "totalTokens": 1261, + }, + }, + expected_litellm_response={ + **BEDROCK_CONVERSE_RESPONSES_RESPONSE, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The user asked to read validation first, though both files could technically be " + "read independently. I'll follow their sequencing and read the validation file " + "first.\n\nNow checking shop/validation.py to see how validate_quantity handles " + "zero, negative, and non-integer inputs, starting with imports and helper functions " + "near the top.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "The user asked to read validation first, though both files could technically " + "be read independently. I'll follow their sequencing and read the validation " + "file first.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "Now checking shop/validation.py to see how validate_quantity handles zero, " + "negative, and non-integer inputs, starting with imports and helper functions " + "near the top.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "", "annotations": []}], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "tooluse_HZpviLzbdTE4T0GMawHsmi", + "name": "read_file", + "type": "function_call", + "id": "fc_tooluse_HZpviLzbdTE4T0GMawHsmi", + "namespace": None, + "status": "completed", + }, + ], + "usage": { + "input_tokens": 857, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 853, + }, + "output_tokens": 404, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 235, "text_tokens": 169}, + "total_tokens": 1261, + "cost": None, + }, + }, +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSzQUKEggSEAEYAjgBQgh0aGlua2luZxIMBQgAH/utL+SlHBltGgxDyOhF0rniv4pJNTQiMNmF1xF0QjswZwV2rMEyccqYlGdGvMcM6/MX" + "lTjznngJK6CmT4WJ0ueKjnCuwChARCroBMQXhDXHvXCbvjESc/c/WnSjWeZtFvycp287jestqbK0BDZ8v9ErwC6iSx2yBNgVFHzpJuOmO06e" + "8Y49ALNuhk9VMl4CGjfbGVdgeGGDrjkk4m9oo0JlYvBykNJiSf6Tu82PuYLgirixNBOf5ZMRX19v6oMZA7RHxJz72XlNAde6VBEjMy6LV/Jy" + "0tcrYI12e9sxjJaDZ4wF2Si6/+VRMRM0QkQHFhVALNFnxvb4kawAPEkx/I1gdBvEB76ACTOjmKYnYdPZPdqu9eScnTamuOGxpet6+82cSPnb" + "IsUXD1XXT0eJUFi40ot60YFcFWTX3vpJYkzMjndouRx998TaLS7TxqUiYFPu2kquJBOY/pDtkh0oysaXBQ0XnZyHtXIRuzFkfWgjkmvPL+UJ" + "ZxDUA0RMuUUM3b6cqjvOgGBg7/1qBFfgc5cr8nPOeec+0E1zk0szyTQ+14fwMMkRte0emSjhPiIIKk2HspOP2IMahYKXx8Ni9xp4WSXwylp6" + "gXyikRTmJRlnknU9NK/VlqJEAcOlZMSPC55ug53ZppDw71ssCiFeCcEGQCMajAbG5DEfLxxY4h4fqGxCqj2qJI8ATA4caGytPY0lBS3+DsrJ" + "sUuzBdVtHiMceMJtEsPVpvAc9ITzl2+rwXiC2+MMeJVXgGpEyCCcksI2Bs2wIoly1/1J9rG9kz71neIhItwY9B3r5LgSEliWYpuSI/LbiZHB" + "9ZwZJj87gKYH/C3U5w8oc+OLZO7zOQ/8bbLIQTyuk+hepc8KZgq5eouSMvhWGbmE+eA+0On+mB+85EmSmFu1JTXt2+D5/BVVbnQYAQ==" +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQS6gUKEwgSEAEYAjgBQgluYXJyYXRpb24SDJFe0B/jn7Vc9l86UxoM+3QEi7NVYrrEAJXWIjDH/Ye9cV6IW0LpbnOf4ZukC5Dm+ug53eFV" + "0Aad2kQp1qLUJ3qu6oXbevVE6AGblSYqhAUYC6yg3pfX1HlZfxFG2YmiNPSo4jxv/nXTib4cG6c64wVIKb7lcnUKVhqsbii2eBZ3+dcyeqah" + "LqgqhS5gz8qM0JWf8erzMW27BwS4z+VflWHyDNcYxZ0N5xg8hxKObWmDNJ1hv46wCUL3BVB2q4IcKMsdxWtIIz6PAllot/4G6ton3kESWcVj" + "k6Q1Moj+RzvX1pDESbIaWry84vTfuH8vq2x5p3yTICKl6+lHFrxt9HWPRW6SebPOJ/7y2Y6qOb6h4bEb8M8sRdm5p/LbhfwBc3DVQ7plPic2" + "IkS7mMwCInmJYbjRdefnUX5fneCwccH56O9IarBZ/s1VM0D/Xv1tgalZb9bNaaw7BvJX/5IkBP6yjYhbtPeh9I/J9XPYeOcNRn1QgbskbEJ1" + "Ff3lvL4jEqydqRyuQcyasCXGxAsFkNqA6grC5sFNBBlk5lcR1T8poXh7d/36BMNTcU5AzEBjzA/i0QQvKwVVw4IkjphHd3r+6ZgRSHGc0m6R" + "WY6NlKfTCQEhF0fZeL77TURzs921CXiynXmSyqvuuEOU6aEy6n56AkjlxdpZ4UgEzGMiij4i21qOpBsS18Fb5lIBC12DqhEXWu/NFthVDlG/" + "7Gwu4o/v1RQIXHaM8Kvxobt75gyaIWp1NIeN9GN/SAiTOjc/zos+HDFzGQYSmV5tPcly/xItwTBDgtJChfujxwd09b6j2WH38Mds89Cdz7CT" + "aplQBwhdVmVTRXhAv9nRDxPWzVnYAtI9hHeLoxU8jJAAXq5f2DErZxF7TdYwR4UGVrj21w+eQ97yPMyPAYP9XbKmo6xp56fMz+TmpBfiNJM9" + "fU+/b+ImXYstTNT/cNu7z7gwZv6fthgB" +) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-opus-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-opus-5-5"}, expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_RESPONSES_SYSTEM, }, mock_provider_response={ - "metrics": {"latencyMs": 10460}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 71}}}, + "metrics": {"latencyMs": 12388}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + "text": "Since the user asked for the validation file first, I'll read " + "shop/validation.py around line 18 before looking at the test file.\n\n", + }, + }, + }, + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + "text": "I'll check `shop/validation.py`, reading lines 1–50 to see " + "`validate_quantity` along with any related imports or constants.\n\n", + }, + }, + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + "name": "read_file", + "toolUseId": "tooluse_ZbXhsiKlmJB473d9A8N5ZI", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 853, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 853, + "cacheWriteInputTokens": 853, + "inputTokens": 4, + "outputTokens": 245, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1102, }, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **BEDROCK_CONVERSE_RESPONSES_RESPONSE, "model": "bedrock/converse/us.anthropic.claude-opus-5-5", - "object": "response", "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Since the user asked for the validation file first, I'll read shop/validation.py " + "around line 18 before looking at the test file.\n\nI'll check " + "`shop/validation.py`, reading lines 1–50 to see `validate_quantity` along " + "with any related imports or constants.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "Since the user asked for the validation file first, I'll read " + "shop/validation.py around line 18 before looking at the test file.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py`, reading lines 1–50 to see " + "`validate_quantity` along with any related imports or constants.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [{"type": "output_text", "text": "", "annotations": []}], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "call_id": "tooluse_ZbXhsiKlmJB473d9A8N5ZI", + "name": "read_file", + "type": "function_call", + "id": "fc_tooluse_ZbXhsiKlmJB473d9A8N5ZI", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 857, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 853, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 245, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 71, "text_tokens": 174}, + "total_tokens": 1102, "cost": None, }, - "user": None, - "store": None, }, ) +CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0: Final = ( + "CAQS4gYKEggSEAEYAjgBQgh0aGlua2luZxIMwBPNPxtNdRTj9z+3GgwSLCcSbOT5eOdg4a0iMMMEuXnI1wytKqyHTo9ejWTxkMM26SCWUqTI" + "H8eIgHNrJHcYTESawe9nmU18j4tufir9Bdw7vFeVYpvoISrtD0A+reIZiKHpRSyvhuL7aiOVQmX8Nm+o9LWnDjMzO8VeHw/JhmEY53nv/kb/" + "aU2CxklgPf+LYWHzlvVUNt+z4vWca/66spf3iduVP1rofkqnKNNSe1q6ECXv9x5/esewnGGjtuxLOB/lTYU3j67KUSGbTmoV9008oLpTnAkC" + "9XOfWprSoh12L0XSqip8rorUlcyD3XLbiVu6MZt+mGbaVv7Qip7tOzOcAwPl/vQKPAFlV7Fwk7Jg/l4vwErw7s+xqANYxZRNPOs/quWe2/tc" + "stL5T7Cm2sGNJx0LI022V9Xfu+MihJvl54c3IWh0duPIU4DToEkNHeRJY++DjrynqXdeWH8eydrtdzbuNhWCGW936UlUKfYoziUhkSS+7MpZ" + "R1hhK7TErXI+5CWbghy7c4KdH8darDfyLCb0KLwYM9A4PqzLL2ZpIZS4oL7EvuXFTC/Wa40Omtmtn4klBwGHmTUHL6JXXaZGXTtz5HqGSaK4" + "8Ia/W+YG+q0IaApw1NSs8MMdnOlt5eF7qVla73EjF8JE3TLYn/NKPqBAdoPKoe+PxxtFY9fkHIHNrnJupT7nt9YR7nf8nzPm6Y7+KeGHkJ6x" + "LHTiv58t2iV2E5jbDP1awcQg4azhpTr/cBz7cyUnTiGJGjwt/NS55rNnfnH24E3F3yUMLGVySc035LaInTJOVTzWfmT9BKmc3T18WQERUEs2" + "SFO6X7BHWV0Dcx4XqedsR//Fjp9v3lspa2fVqd5/+g4xq8z2kBZlBnXkLQMFNpFwd4ZRiVzRx94u3AzSza2fHFByDhLSHkaQpblg79lo1JDG" + "Op2fbO0TZnwwUsth4Qq/9QvvtpYtAEMox9ISsqMGCELZeT8StjFfAmro7iwp2nGmVfGfjvoTpOZ8oaQChOdi7V/cy7An1v/58GMapRqEKWCj" + "JtuI5qNyka4ECp7FdT++oQJeU7U85PQurp55wlEf4drBvjYl/aICnnxH6ueg8izsmC/hya5IRdRfD7dQoxgB" +) CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/converse/us.anthropic.claude-fable-5-1", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/converse/us.anthropic.claude-fable-5-1"}, expected_provider_endpoint="/model/us.anthropic.claude-fable-5-1/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], - "inferenceConfig": {"maxTokens": 64}, - "system": [{"text": "You are a terse assistant."}], + **BEDROCK_CONVERSE_REQUEST, + "system": BEDROCK_CONVERSE_RESPONSES_SYSTEM, + "toolConfig": { + "tools": [ + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + "name": "search_files", + "description": "Search source and test files for a literal term.", + }, + }, + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + "name": "read_file", + "description": "Read a range of lines from a project file.", + }, + }, + ], + }, }, mock_provider_response={ - "metrics": {"latencyMs": 2671}, - "output": {"message": {"content": [{"text": "Hello."}], "role": "assistant"}}, - "stopReason": "end_turn", + **BEDROCK_CONVERSE_RESPONSE, + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 102}}}, + "metrics": {"latencyMs": 9400}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + "text": "The user wants validation read first even though the files are " + "independent, so I'll respect that order: read validation.py now, then the " + "test file next.\n\n", + }, + }, + }, + { + "text": "I'll read `shop/validation.py` around `validate_quantity` to see how zero, " + "negative, and non-integer values are treated; the test file comes next.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "name": "read_file", + "toolUseId": "tooluse_IZcZuTirqaML2t3hxtMt9K", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, "usage": { + "cacheDetails": [{"inputTokens": 873, "ttl": "5m"}], "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, - "cacheWriteInputTokens": 0, - "inputTokens": 23, - "outputTokens": 6, + "cacheWriteInputTokenCount": 873, + "cacheWriteInputTokens": 873, + "inputTokens": 4, + "outputTokens": 250, "serverToolUsage": {}, - "totalTokens": 29, + "totalTokens": 1127, }, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **BEDROCK_CONVERSE_RESPONSES_RESPONSE, "model": "bedrock/converse/us.anthropic.claude-fable-5-1", - "object": "response", "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The user wants validation read first even though the files are independent, so " + "I'll respect that order: read validation.py now, then the test file next.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "The user wants validation read first even though the files are independent, so " + "I'll respect that order: read validation.py now, then the test file next.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "I'll read `shop/validation.py` around `validate_quantity` to see how zero, " + "negative, and non-integer values are treated; the test file comes next.", + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "call_id": "tooluse_IZcZuTirqaML2t3hxtMt9K", + "name": "read_file", + "type": "function_call", + "id": "fc_tooluse_IZcZuTirqaML2t3hxtMt9K", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 250, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 102, "text_tokens": 148}, + "total_tokens": 1127, "cost": None, }, - "user": None, - "store": None, }, ) diff --git a/tests/integration/translation/responses/bases/bedrock_invoke.py b/tests/integration/translation/responses/bases/bedrock_invoke.py index 7885e05b3a6..c5093f9c5f0 100644 --- a/tests/integration/translation/responses/bases/bedrock_invoke.py +++ b/tests/integration/translation/responses/bases/bedrock_invoke.py @@ -1,7 +1,14 @@ +import json from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import RESPONSES_REQUEST, RESPONSES_RESPONSE +from integration.translation.templates.bedrock_invoke import ( + BEDROCK_INVOKE_REQUEST, + BEDROCK_INVOKE_RESPONSE, +) +from pydantic import JsonValue CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -92,97 +99,6 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/invoke", - expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - }, - mock_provider_response={ - "model": "claude-haiku-5-5", - "id": "msg_bdrk_prp4tfjg2ibow6km7wkqw5euejxxssl2ngm6wbgjxyg7xd32tkja", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - }, - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 23, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 23, - "video_tokens": None, - "cache_write_tokens": 0, - }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, -) - - CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", @@ -452,273 +368,602 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( +BEDROCK_INVOKE_RESPONSES_SYSTEM: Final[JsonValue] = [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "responses client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", + }, +] + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSuQUKEggSEAEYAjgBQgh0aGlua2luZxIMkzWpvtxVqloRdTejGgySvyYIIigSiTigYwsiMOl8XJI9rkOBrw8tVJltn2ZOQqbsCTr8UQ1o" + "5xBXnkf4Pj+F0fd7GRGRGWVhSLbFCSrUBMeafwRFgNCuZyPdn7n38LWOXm7+Lg8tjM0m2PfA04geMhUvcih7RPypybFHnZgmoxqTnoxhyMYA" + "BJ6CuZ1PHCKCXXTLGF2SsBJCCKhOeDwlBnVbwgbxvTUScOtpx/12go2VloE1aRsFTdpG4oZlziwPqV8GYUv0Pzw8zlEAbdlK/A/pu7vH1eM+" + "08tFzjRwl5/ACM8NIQ3s/baA58Y1Cfu2yTUyzxV13+MaBnGEhEMPhI/3muGRHhT20ga9jOhGz4uH67he+BTschcNoIi6Pw2ZUy6PAnfjs9Sx" + "3tJMQyUCThBsmKS1nGSTu4R8NQMKJWnMGr0X6fyk5PphRmC4YEYTjJ4nw3Ad6Kog06Bh1LbJTphVX3V3qc4sMBbrKSPH/vd/f4OCgrWU572B" + "lxYl3HNT+gwJa+uBfQShW11KQR8A0v8T33jnHAmysQ87+iJpTm1u+W3oMoemDsUddpICg6JQdxDSA4Ft2iuDds3XgeeOmXYl8mc8pYzuep40" + "zUjmyJ9cPZYjWNArd1IQAFk6s5wa47jHxP72gXiiXDJabXGT1/xvsafaM9WjZ/cxmmyF2K3122br9OWknau/rJa0gg1+PluGe/vUptHi0Acx" + "a4pNbvcQFiRutnwjPr9uqQMqk+cqRdBVS2pZAc/vnmGi7m02MjBuHRqZeFMIkc4WYK3fgt0BxnMyXMJCzQcx64oGg5ZgU15crKV/thvRkvsI" + "AjIl4ScwNIsYlQcz3fx7c1yH2hRIWb+ahmdGbJvnDlcNmliO50qWyhRyzYBoCU55cvXtnJTHGAE=" +) + +CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/invoke", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-haiku-5-5/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", + **BEDROCK_INVOKE_REQUEST, + "system": BEDROCK_INVOKE_RESPONSES_SYSTEM, }, mock_provider_response={ - "model": "claude-sonnet-5-5", - "id": "msg_bdrk_tozg7mwrkjb7fisuua7q5ylekeypufs6he64a2tlncr4bvdg4dha", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + **BEDROCK_INVOKE_RESPONSE, + "id": "msg_bdrk_ytkmzvqioxcm74hxqxgoycuu6vt3orsozxii4miswb5tk57ya2ra", + "content": [ + { + "type": "thinking", + "thinking": "I'll open shop/validation.py to check how it handles zero, negative, and non-integer " + "inputs before deciding what to look at next, starting with the first chunk of lines.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01Uivu38DoXo46h5UDP3hKz9", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 175, + "output_tokens_details": {"thinking_tokens": 75}, "service_tier": "standard", }, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", - "object": "response", + **RESPONSES_RESPONSE, + "model": "bedrock/invoke/us.anthropic.claude-haiku-5-5", "output": [ { - "type": "message", + "type": "reasoning", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "I'll open shop/validation.py to check how it handles zero, negative, and " + "non-integer inputs before deciding what to look at next, starting with the first " + "chunk of lines.\n\n", + "annotations": [], + }, + ], "phase": None, - } + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "I'll open shop/validation.py to check how it handles zero, negative, and " + "non-integer inputs before deciding what to look at next, starting with the " + "first chunk of lines.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_bdrk_01Uivu38DoXo46h5UDP3hKz9", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_bdrk_01Uivu38DoXo46h5UDP3hKz9", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 175, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 75, "text_tokens": 100}, + "total_tokens": 1052, "cost": None, }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll open shop/validation.py to check how it handles zero, negative, and non-integer " + "inputs before deciding what to look at next, starting with the first chunk of lines.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + ], + }, }, ) +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSiwoKEggSEAEYAjgBQgh0aGlua2luZxIMOIdSxTR4GjBHazgyGgzAFsCOD/EpsKJodeAiMLAgKd6UJh7VQCix7AFKFkbf6rPK16fItxJ9" + "PI5nOnHUS0SfBCxVY4LZmRg71SmHYiqmCTfFsnMccIbkeJS2+Xap1iQw/HZ7KYHJJsCqDxm9Ua2I8LHYyL3s2OVT9NzUiCMKZQWPQ2DqBWZ0" + "gsnrDQ9ZYkNrtXWxypuIDhHQw7dOs89033JTcFGdnxpFwvpvARtjSHPS7O9UOY48wLeD+K0AL6bzJlmbarntN3xeqYnYFU9cAIhyjSrCYWtE" + "5Y37pblxLuwXjHI20rSyT2carPP7whuU2LDg8GVIu0JdDO4seHyys9s+BaiMJEPnn2naeK3WAyOvM5wTyvYj81cxx9xJBlVuFyeyxxyTQgnL" + "DyM7OtCiy35affvC1oKXwYQyiRUYso/4IuEBTiziluiO+wYg/NAHUycI+HEdFp7zOIK9AdC1WtB5xbUtclcCObYpPgdSfR8RtcgaUpltqIfJ" + "VA+9K+hDDNTKKh/Xa4+WFvVj++6NHe7glBlQeasGOtNtrFbRJUB6dgnorTWblMq8NyzCASK5FoQo77Fv3lKdAleW9ujj1eGA2hon5UoO9BGQ" + "w0pnLzColC0V3NtAbpuC8Jl9mm5te2ChvJJV7bYa6/3RYlA3fRTAvP6grSCRTMYkxnqtJCliTb6sjly99vkD4zV7gMvjY8BW08rVruhMShUk" + "gHRC6XBMw6kvUboxDnMs1yCJCYyHqUI1AHaP2g24VigzgWwOPpNSRm1lgodCYZUM8/gznj1JuAwe2XoBXgfBvkRQCYXorCulmvjK3NMIBzdU" + "fsfyGExkFdaQUJLbhWq6IF6L0b9iKLuugkavPUaodPpsg6L3Z1XHsFDZgO7GxRoHcKvV1/cEtO6DU7cc+gu4D/dSsnp8K2BY00UKfDyBqG90" + "OwLw6IDyaiw829KOc+3lVBW0vwxhbA3AV++soacROCplQ/Uy9qQy8lB63whvFu4wGypT6yn+2k9TUSNSbgimfvO3SzuWUenEpnkSVO8CFvRQ" + "iffW03nhllZyHHlwB9B+NSrghpom2VvAd4OBg7J4353Favtsv/nu6LRpUUalVbkmX37iwZ/Q8gEYooxqa/EZodr0+1mwHvRJiascYs67I+MV" + "+9EMffg2inNTIKfDu5GzRJOKWULP4e60/9Q5oJYZ5Pw+LkUyrIQphe+jNAczMq4N/TBPVIQhJWgJaCNfSV3JGvvLPOgPEtZjaIQHYIVn+sID" + "XH/+DsPUWSLKDLibTDPAoCgIQQN2U2TJLEzrs4fr1wej4fJlQ7ufZRXcRisAmbsM4AUqRXrErjjcfgg2cgtPN3Ol1Yfg7XNm5fw09WJD/DKC" + "C34ocjCqBXsZDQtb8T3w54OxIiRLn9VMj1pizbgC+x2htmyBt6BNHns7K08DM67SiG7ph049Ze/SFoZqb4waCUu1dRYZAUKUo50Uy25Xi5Qi" + "cHhcUOWHxIiTl50THEQCA884azGo9ZNpby+krX1dfVuGSovwvESk0GGNM+LgKNw1tpj2srpQZIsMG8Oo1fFxriIX5dCJq1v4J8gHw75q38wx" + "ySwPds/XHbkgK3KgkVrL/XcfUP3myMke3shNr8o7tm8SulaUZn0J4hWawGegO6wS/44Q76OoeLluOvOHyERd6wMPquKzo9lZqATi0x8CrpC5" + "GAE=" +) + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQSwwYKEwgSEAEYAjgBQgluYXJyYXRpb24SDHTsA0CfvetGZFL8axoM4khrNWxDqnQ4bENWIjDrerGCpIXFI6LZF2Hk4OYvtcRO6Ytxz5kc" + "E7vKFBT57mJF6FitCHcxRNk1M7/C2L8q3QUy4sf0KVQTWQbGHFutqk/OUkcyJvaS/tPjOrhqUWhsWRsInGBGX3sjSLMWWNQ3bHdnql0bVDFc" + "YgLfO2K+1Ta2P3pna3jpvBO+BM+W1xQ1FrenEWqbXtNwueTC6kQqGUZnmULo3cbMb5l1Yo2nui6i87a3Bs03jtomyzjRb+xMH2XNpePzURLf" + "ADvZIHyAPOlHOnK6ZTk4rugCEHHTAudo9Hz/6ZDTjXTpqzDD69lxgcdviZ2oNMpExhPxJctgzNBuYdFYjT09B0Atag0HLXYdUL6L5Cz1599n" + "AiTQl60CWALeyuJS65GikY3FNn5tXt1IPHLUsNsoEzBr07lvZ1WeyyiyoExBOCSgH0KuxSIP9TL4hQsMiI9kduv3w4J9ybSyViI4sXg3nlUz" + "5UqNonSx9487qwcmlZxZQquU85UtyA6Kcym6oAUH3cr1+VrYvpBZzwku00ccQQ2faJGZ18AIBYwvY+HZQ0bPop7AkYSqIyeLe5A8vcm3O+o1" + "sRpj05W9C649sYuPo9qXUabkB/WlkoLydG3X+erc1ry3c6yur9XYrtGfXcVCdA2xtYOkzYlBXrYTbVL/oes3aBTt2Yg+0OLaa8XM7rvajpF0" + "QkYwtAQwWwyBe2gPtMWVJUuvljICUwMhKPz2bD3/7X/cuNKmso3Y+sGzoCtylrggSkdVdMNB1zTebX5iPmIk2LDGuEDyqyiJGugGo04yEzyv" + "Q1icaJmssvQprgCmJJkHB1PnR3Lw2tMwCFgyrJxXLi8lDSyFmcl9YoHF0OBcw0+HFAuZsgkbjHXeVU/K3jFAj+GP3a7HBrwoN2jBHI9RfU74" + "Un4iBaoK8JWdy+/vR0vdGiifISnlzmSOL9y+mmJubPkLMxwH4+vnfXuDMYvDhgn1M7vSUwiHePupdVz6HxS2LjY7XrwVgLSZHoGmvdP6NoeB" + "kwzjb7hXaitkAzyqZls63keq+Hw+tPTlMGO5NtPeGAE=" +) + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/invoke", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + **BEDROCK_INVOKE_REQUEST, + "system": BEDROCK_INVOKE_RESPONSES_SYSTEM, + }, + mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, + "model": "claude-sonnet-5-5", + "id": "msg_bdrk_i3avv2mb3domrfdrl6s7b5mubzt6vckxqeea6ieovrsgmyzby7ca", + "content": [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, though reading the test file in parallel " + "wouldn't conflict with that instruction. I'll prioritize following the explicit order and " + "read the validation file first.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll open `shop/validation.py` to see how `validate_quantity` handles zero, negative, and " + "non-integer inputs, along with `tests/test_validation.py` to check the expected " + "zero-quantity behavior.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01FfS5J9LzLxLwq5caHEM4xW", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01FTMGZf7C2vzjc2Ux2SJyoU", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 536, + "output_tokens_details": {"thinking_tokens": 242}, + "service_tier": "standard", + }, + }, + expected_litellm_response={ + **RESPONSES_RESPONSE, + "model": "bedrock/invoke/us.anthropic.claude-sonnet-5-5", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The user asked to read the validation file first, though reading the test file in " + "parallel wouldn't conflict with that instruction. I'll prioritize following the " + "explicit order and read the validation file first.\n\nI'll open " + "`shop/validation.py` to see how `validate_quantity` handles zero, negative, and " + "non-integer inputs, along with `tests/test_validation.py` to check the expected " + "zero-quantity behavior.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, though reading the test file " + "in parallel wouldn't conflict with that instruction. I'll prioritize following " + "the explicit order and read the validation file first.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll open `shop/validation.py` to see how `validate_quantity` handles zero, " + "negative, and non-integer inputs, along with `tests/test_validation.py` to " + "check the expected zero-quantity behavior.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_bdrk_01FfS5J9LzLxLwq5caHEM4xW", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_bdrk_01FfS5J9LzLxLwq5caHEM4xW", + "namespace": None, + "status": "completed", + }, + { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_bdrk_01FTMGZf7C2vzjc2Ux2SJyoU", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_bdrk_01FTMGZf7C2vzjc2Ux2SJyoU", + "namespace": None, + "status": "completed", + }, + ], + "usage": { + "input_tokens": 877, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 873, + }, + "output_tokens": 536, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 242, "text_tokens": 294}, + "total_tokens": 1413, + "cost": None, + }, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, though reading the test file in " + "parallel wouldn't conflict with that instruction. I'll prioritize following the " + "explicit order and read the validation file first.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll open `shop/validation.py` to see how `validate_quantity` handles zero, negative, " + "and non-integer inputs, along with `tests/test_validation.py` to check the expected " + "zero-quantity behavior.\n\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + ], + }, + }, +) + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSgwUKEggSEAEYAjgBQgh0aGlua2luZxIMT1FeqW5PaXtFp114GgyoxyN+NbjKsT1yxB4iMF13gzAPt/OOukSm+Is78v7dbu1WcrJvArvI" + "HMK3ZeviwjQPyQ0zywjsqDWnepR7/iqeBAFSYFwVgmpUGxZIoN7tQ+XnRVF+YNYPY9HbKw6lE89F3jejjSkPFYWIUkfW2KsB3/kEmkWrXzV7" + "dIWAxKfGGI6gvIvzVGl8UoKB7cYJqtx1888mqoLH8mGjYRtMrPang71iBUoiJjThOdSiPSTr+HnvGb6e9CpXduwBsuCATo+ZKAKiyDoYEqr9" + "HRmvz7fi6RuKSErHfG9Zod7muUKjqw2Y/kaVJenornm8JuMYuYIfnv0II3MBHtH0nKr6ZyPuaALFl+L7sh3eazeahWzC3CeZ8LZmLp5NuOg+" + "xsHNJQDjWJuXIUrCoYdc3gnCgcnuEYd2dz2+tmrR7K509+uCOe9EEoO1SuKU+YM9qkSBVPo3VR99cf2XpUc3K+4v1KLocUjtLlgoLmxlpES6" + "uJLhyP65XlvIFAOGAYH0hiwvbUq55i36T9LHG4mbUdwzjum1CSzZ8trx3k1lzcsxlLVPIY3JuBE63HfqWZs/T8UobfsPbTCrMgBqpmR+wBdi" + "8wKJPiasyregTVYWuhMrg1LFodr80/co5TLeLAvoI/oXs6Syz8ME8IMpKD4iHNr7Pj7/Le/XEAU6W+2n/b8Hx6NVRysrl729eIqV4Y2pk2Xl" + "CUCVx0wfuAlvw5TR4Ft6Eb7/wbSOlxU/E5iqaJusZbpQLEtCTUXU4IKgmaPJZFxKWIEv3hbROZ477bvTyROQgaDlcabkiPH1o6FREM5BeGI1" + "GAE=" +) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-opus-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-opus-5-5"}, expected_provider_endpoint="/model/us.anthropic.claude-opus-5-5/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", + **BEDROCK_INVOKE_REQUEST, + "system": BEDROCK_INVOKE_RESPONSES_SYSTEM, }, mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, "model": "claude-opus-5-5", - "id": "msg_bdrk_mxljoibckai4wqyt4wd7cmslqjtlkals5cbwms2hackerasjglha", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_cwcayjjtio2jq5g6a45acoimfxqughi7fphihxe5toxmhqojwboq", + "content": [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py near line 18 first, as the user asked, before looking at the " + "test file.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` around `validate_quantity` (line 18) to see how it handles " + "zero, negative and non-integer values.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01J5u6tgn3ppV4c2VpmD2xy6", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 196, + "output_tokens_details": {"thinking_tokens": 51}, "service_tier": "standard", }, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "bedrock/invoke/us.anthropic.claude-opus-5-5", - "object": "response", "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I'll read shop/validation.py near line 18 first, as the user asked, before looking " + "at the test file.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py near line 18 first, as the user asked, before " + "looking at the test file.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "I'll read `shop/validation.py` around `validate_quantity` (line 18) to see how it " + "handles zero, negative and non-integer values.", + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "call_id": "toolu_bdrk_01J5u6tgn3ppV4c2VpmD2xy6", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_bdrk_01J5u6tgn3ppV4c2VpmD2xy6", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 196, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 51, "text_tokens": 145}, + "total_tokens": 1073, "cost": None, }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py near line 18 first, as the user asked, before looking at " + "the test file.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + }, }, ) +CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0: Final = ( + "CAQSxgcKEggSEAEYAjgBQgh0aGlua2luZxIM93CYh6+fYNYfl0lvGgw7Sq5PzeOUQaR9JKoiMK9/V24UwNwvJbJdQjDEFCLEIn3lZgk7//Uw" + "Hr6kTvOTtxS6Xdgt1IphkLtCfI1rWCrhBvKpjQHMVva70EZuEDWg2HPEMrmnrvUOXJniTRUiArTXQTR6itTG2XXqxHHsARUCq/1ECbBG9j5X" + "Ti6iF/bgGEVkeWr4rmCNYnQtGS1Rn+f83bHPbnBwyMOH29aX0JS9kstQFoaPOa4G4BnzVgFg5Tnj6JN9hh+aSoBMCliraNTo1lirHHpQXqCZ" + "jupkIKRjDyvRfpE96tGjtWTJFJ3f8HTm2PiTGUOhCz7PoypLgCfOQWyfaptuIUKa9sZXtJVlLBmoj1XDqIqQKsCcvDKPjDnAVVsTr4dAExxP" + "g/3ujwP5Y/+LDHFE9XPetApyfiIh3qy5L9UH1Yv9RwKLqfn3KWPlXEWoCrvgOTj7e5EfTk7Cu3WxDNp8SNs0TeFnGsCKQGASuqTtgZTYzrpD" + "Jc1EbTZOt+CRVUgVhcbadMSdhpy/PcAtgiZ8/MvZCCv7BeORzmVYb9cYaVFVQK4/tchF/lcK+4YVvDHCCSXAwhj6Y9NyQ7j94JmFNqhjsSK4" + "t4qALhhmOcJuCispP6ZRhf1SCoajBsRVRdtvPbFnOH1GqdErrzAEcD9VZk1mkmUN6fs9Vl9Uva6XjVr7YzQHjdkGZ5fEHcIpmJrS1ZMI+YuC" + "quoy4qMJvp8g8YgoLYjt6GFWqwOwePTuFmMnL7VadlmEtwAionrUqEXCwSh8FVOeN8mvaoA2V2AtGp5zA4SDmbrF+XhuNS77ac0dCcQbs8iy" + "H1a1oSySzqGf+muJBkoSCVtIAPQ48+9ha3fGM49fj2EQINCu4NhcBylZqFhHV7FJA620qqEJV8kWBJqc6j9pYHQYSirhnBLxPFPFQS3UNItx" + "k4c5oPs3VGPea++KQhhbRnlAEPu1pKzKMM7yg9+rT28l2XiIGnWSqBo/J8ipsL/uFcthzptNtoK75pxhGduvynAI7wRbArxT6UMegIsPdRlH" + "uXBbyXKuXhbnCoCkbb7VNbsO2tWWqE5rUdLgKyHd57SL3BlJDVd+V+/bW84zqP7QiSIMmrv3Q8lVFjFyOEXfeRZF7pXapgdAK5CWgoKXwdVF" + "oar+HuszSP6LPqJggfD+9eqZhpZI6wmUzSVc9TlZAV+fB7pWcDVgBYG2Vk6bckiXOZrJigDuAbHObeZ6BcKQpID/9TvvtNKnQGJy4Uva2p8Y" + "AQ==" +) CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock/invoke/us.anthropic.claude-fable-5-1", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "bedrock/invoke/us.anthropic.claude-fable-5-1"}, expected_provider_endpoint="/model/us.anthropic.claude-fable-5-1/invoke", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", + **BEDROCK_INVOKE_REQUEST, + "system": BEDROCK_INVOKE_RESPONSES_SYSTEM, }, mock_provider_response={ + **BEDROCK_INVOKE_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_bdrk_kct3wlyhliymwyvqmjn6uy72qpg4hekwep4upsfqdec3b7wqf5vq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_bdrk_ehpngdq5umo3xytr46gfdb3ifglm2bsofgtx5zargdynfbgdvepa", + "content": [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, so I'll follow that order and read " + "shop/validation.py before moving on to the test file.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` around `validate_quantity` to see how zero, negative, and " + "non-integer values are handled.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01Ee5Sz4v3R2bgHS6gQV5m8N", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + ], "usage": { - "input_tokens": 23, + "input_tokens": 4, "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, + "cache_read_input_tokens": 873, "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "output_tokens": 283, + "output_tokens_details": {"thinking_tokens": 142}, "service_tier": "standard", }, }, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "bedrock/invoke/us.anthropic.claude-fable-5-1", - "object": "response", "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The user asked to read the validation file first, so I'll follow that order and " + "read shop/validation.py before moving on to the test file.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, so I'll follow that order " + "and read shop/validation.py before moving on to the test file.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": "I'll read `shop/validation.py` around `validate_quantity` to see how zero, " + "negative, and non-integer values are handled.", + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_bdrk_01Ee5Sz4v3R2bgHS6gQV5m8N", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_bdrk_01Ee5Sz4v3R2bgHS6gQV5m8N", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, - "cached_tokens": 0, + "cached_tokens": 873, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 283, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 142, "text_tokens": 141}, + "total_tokens": 1160, "cost": None, }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, so I'll follow that order and read " + "shop/validation.py before moving on to the test file.\n\n", + "signature": CLAUDE_FABLE_5_1_THINKING_SIGNATURE_0, + }, + ], + }, }, ) diff --git a/tests/integration/translation/responses/bases/bedrock_mantle.py b/tests/integration/translation/responses/bases/bedrock_mantle.py index 6864930178d..c5f3ebaa13d 100644 --- a/tests/integration/translation/responses/bases/bedrock_mantle.py +++ b/tests/integration/translation/responses/bases/bedrock_mantle.py @@ -1,8 +1,17 @@ +import json +from collections.abc import Mapping from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase - +from integration.translation.responses.templates import RESPONSES_REQUEST, RESPONSES_RESPONSE +from integration.translation.templates.bedrock_mantle import ( + BEDROCK_MANTLE_ANTHROPIC_MESSAGES_REQUEST, + BEDROCK_MANTLE_ANTHROPIC_MESSAGES_RESPONSE, + BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, +) +from pydantic import JsonValue CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -193,101 +202,6 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( ) -CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock_mantle/anthropic.claude-opus-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/anthropic/v1/messages", - expected_provider_headers={ - "content-type": "application/json", - "authorization": "Bearer synthetic-bedrock-mantle-key", - }, - expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - "anthropic_version": "bedrock-2023-05-31", - "model": "anthropic.claude-opus-5-5", - }, - mock_provider_response={ - "model": "claude-opus-5-5", - "id": "msg_bdrk_tcemmkaonjjx6pb7gr76umjdqmuwgte56kvf6itwjylegy4lpc3a", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - }, - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "bedrock_mantle/anthropic.claude-opus-5-5", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 23, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 23, - "video_tokens": None, - "cache_write_tokens": 0, - }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, -) - - GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", @@ -698,249 +612,352 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) +BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in RESPONSES_RESPONSE.items() if key not in ("provider_specific_fields",) +} -GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( +BEDROCK_MANTLE_ANTHROPIC_MESSAGES_RESPONSES_SYSTEM: Final[JsonValue] = [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "responses client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", + }, +] + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSiwUKEggSEAEYAjgBQgh0aGlua2luZxIMOOtS7Hiav56KWragGgwpR/YfU3idr9yBpQMiMK4FkesXYYneMZORQdVYD9RW4aipV/xs5/8Y" + "7VLXJoGMgJZTAbfRIx1QTTsOahmlTyqmBHNIuHNZzm1tjsg4KBs2SM59oBG0Y1rzAq60RhMBN56yA7XQxO7tYyklXQlMlvqywK/CM2LopG/i" + "Grcau7JI/jEEu4bMkHn0mdyWlg2c/XhK+NcXJDQwsP0fFW3+I/QftObPLaiblgU3LldmQdnut2da7Rn+VoufrkBbHP3GHKVo3HpIN73s4juH" + "riD9mPpg48Q4DY/u3HdNiHJkfq5tDNFS2Fgpu06Y0tndoeUQWoHuPx7NqJ0lqK4+zTvFx6bQ3rzqsBZJqI2gHocRYdtCiULDyC67SOLtFfXr" + "LSZS4lMFJrS6RMBqdd4edAwh6LkrJ1ItbfM85OvI7iRQOp82+UEXXncJgSCPo2Th8RuBdzo/zQswj5L2KMbXdkkNEEGXexW/YccG7XhOj9XM" + "V5SinZlJ3QF6EbgdyVbukyG2yYUrxqjAMMpR+upoHmjWs+lqm3EehyO1CldU0COcvTNP/3vy19/c0ImyJ0bc/uki5Xtz+9sqpd+gZ0gCf5Nc" + "pLh4grY1U3IgEdZEKlUzO2T8OuKzgxBn6u+z0mqB1ZWARgx+R0HKIy0Xyly7gNaikyZwXUCKkSepOnb2xeUMwQ5t4qnAMBLjWnILxnnYF4Jf" + "EVD2N7RhobOXLhl+izVYoB2tQBiKfR2+TloExvsWWT05YQM0uLSUxp8sAJHQgdm0vywVaQlmZ7LpYYs1pKLnsI5XNT/G+f4k8+URqoJ2RcTm" + "BFSkg7bYePEYAQ==" +) + +CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock_mantle/openai.gpt-6-luna", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/openai/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock_mantle/anthropic.claude-opus-5-5"}, + expected_provider_endpoint="/anthropic/v1/messages", expected_provider_headers={ - "authorization": "Bearer synthetic-bedrock-mantle-key", "content-type": "application/json", + "authorization": "Bearer synthetic-bedrock-mantle-key", }, expected_provider_request={ - "model": "openai.gpt-6-luna", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + **BEDROCK_MANTLE_ANTHROPIC_MESSAGES_REQUEST, + "system": BEDROCK_MANTLE_ANTHROPIC_MESSAGES_RESPONSES_SYSTEM, }, mock_provider_response={ - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791240610, - "created_at": 1791240610, - "error": None, - "frequency_penalty": 0.0, - "id": "resp_cazpgby3iulwndtqlxaczoyacnql23jugwncarclttip7ewos6na", - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, - "metadata": {}, - "model": "openai.gpt-6-luna", - "object": "response", - "output": [ + **BEDROCK_MANTLE_ANTHROPIC_MESSAGES_RESPONSE, + "id": "msg_bdrk_vtj2w754torwaf46a3regz6cuegn44qhpglkshgsojppqmroreza", + "content": [ { - "content": [{"annotations": [], "logprobs": [], "text": "Hello!", "type": "output_text"}], - "id": "msg_f693bb420fad55468f26ef3b43e761fd", - "phase": "final_answer", - "role": "assistant", - "status": "completed", - "type": "message", - } + "type": "thinking", + "thinking": "I'll read shop/validation.py around line 18 first, as the user requested, before looking " + "at anything else.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "Next I need `shop/validation.py`, starting at `validate_quantity` on line 18. That's where " + "we can see how zero, negative and non-integer quantities are handled.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_015nyupS1RboZpDnHW42rzbD", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "in_memory", - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "safety_identifier": None, - "service_tier": "default", - "status": "completed", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", "usage": { - "input_tokens": 19, - "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 873, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 207, + "output_tokens_details": {"thinking_tokens": 53}, + "service_tier": "standard", }, - "user": None, - "moderation": None, }, expected_litellm_response={ - "id": ANY, - "created_at": 1791240610, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "bedrock_mantle/openai.gpt-6-luna", - "object": "response", + **RESPONSES_RESPONSE, + "model": "bedrock_mantle/anthropic.claude-opus-5-5", "output": [ { - "id": "msg_f693bb420fad55468f26ef3b43e761fd", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], - "role": "assistant", + "type": "reasoning", + "id": ANY, "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I'll read shop/validation.py around line 18 first, as the user requested, before " + "looking at anything else.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py around line 18 first, as the user requested, " + "before looking at anything else.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, + { "type": "message", - "phase": "final_answer", - } + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Next I need `shop/validation.py`, starting at `validate_quantity` on line 18. " + "That's where we can see how zero, negative and non-integer quantities are handled.", + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_bdrk_015nyupS1RboZpDnHW42rzbD", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_bdrk_015nyupS1RboZpDnHW42rzbD", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": True, - "temperature": 1.0, - "tool_choice": "auto", - "tools": [], - "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "status": "completed", - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, - "cached_tokens": 0, + "cached_tokens": 873, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": None, + "text_tokens": 4, "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 207, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 53, "text_tokens": 154}, + "total_tokens": 1084, "cost": None, }, - "user": None, - "store": True, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791240610, - "frequency_penalty": 0.0, - "max_tool_calls": None, - "presence_penalty": 0.0, - "prompt_cache_key": None, - "prompt_cache_retention": "in_memory", - "safety_identifier": None, - "service_tier": "default", - "top_logprobs": 0, - "moderation": None, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py around line 18 first, as the user requested, before " + "looking at anything else.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + ], + }, }, ) +BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_INPUT: Final[JsonValue] = [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + }, + ], + }, + { + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", "line": 7, "text": ' + '"def test_zero_quantity():"}]}', + }, + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Read the validation file first so we can understand how zero and negative quantities are " + "handled.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + ], + }, +] + GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "bedrock_mantle/openai.gpt-6.1-sol", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "bedrock_mantle/openai.gpt-6.1-sol"}, expected_provider_endpoint="/openai/v1/responses", expected_provider_headers={ "authorization": "Bearer synthetic-bedrock-mantle-key", "content-type": "application/json", }, expected_provider_request={ - "model": "openai.gpt-6.1-sol", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "input": BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_INPUT, + "instructions": RESPONSES_REQUEST["instructions"], + "tools": RESPONSES_REQUEST["tools"], }, mock_provider_response={ - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791240614, - "created_at": 1791240613, - "error": None, - "frequency_penalty": 0.0, - "id": "resp_gt7xddo5iwmhm7nsml7dsko5lwptca2telb6kmivpwlm7hvjospq", - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, - "metadata": {}, - "model": "openai.gpt-6.1-sol", - "object": "response", + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027277, + "created_at": 1792027275, + "id": "resp_ncuf2qbhdiy3ykbk6y6t4iduokpwdg2h7xktdw65x3azqiv7xxbq", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "responses client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", "output": [ { - "content": [{"annotations": [], "logprobs": [], "text": "Hello!", "type": "output_text"}], - "id": "msg_b6510810920854d29a290136d5eb08b1", - "phase": "final_answer", + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeGleCFHR3X7I/3TQIxhAYfW9mjX0cFKFSki7Gf5uzENSAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBMTVQNEFLb1hPRHNZMHUrbGhkYnExVzIveHBJMllVbENGQm5BY0RxMEVNZnh3c3BUS1NMV2t5N2o5SGRON2RuU1E9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcG493+ZcD13Vo9Z095jldqzOPdER/g3kHez3TwwTyexPQO0PepR8jRQ3zVYQVnB+5HsLoOUbfxtpDKFNZdhkG7vX7H79h6S2p5Tmcda0OnUhLEa1gj/FATh1bTrkCAAAQANAHetSqAaabgN4pn8RLag9s76ySEeodJgSjG7iLEOmNhVDDpO42M0mz4lH1lu3AXf////8AAAABAAAAAAAAAAAAAAABAAABw/bggpCep0rKr5bUUaXRidg+rbrNYiSeN0LEjiXKzD7Wt/+NhdY6fopjNz8a882sSCQGS6uQbFLsNO+b8+enVXmNgn96/OH9Nh89uQrTE/oPwYWD49Ts+5h4TJqljt1QVto3/i1qiVErPLn+0jpC8qZZKgOsUx8kApialKG9iJdnxT7xWsSSNALi5rt9ckiDVtw7HDLgzqIWs5d5kMBNj11466qX8pb+TYkQIAAFoR/Crx8KG3YqoX5kZJDZrUNFG0mz7arLDO9UaYG/ntFSqDDJmsYlcpyJchpzMZBNrO+5pQ4G8DIdBosJNnR/r5ebFJX7aLT0LWM5wI1morAlmNlhe+TOHPgY6AHpRuHu9cj1EWbuj1LKx+XQAu5NUEjIKFOFs+tVdENroClJnLtP1P4Ncn9DVjdy9pKzy6rt3UdpFkyIHizfPIHA9Ep6JEU37r2lzrJb2Bn9JtwRKxY7gB8TxUk/5tZ5DqzbPc+0KfFM9D1RL06U+birMIJwC8AM9zWpl0basljGg6Wg3t2Ey4Gua6M+gd5sGF0wQtDRShPH0VEy0dl6ZdfC8G0b1qxGWPpymastGVvw2CamaGJEuHexPr3//CytpLZYZmn/aIemUxH1AGcwZQIwF1zAfYPc+CW98SZN8eOtgH6Zvjg9iy37mmtat8ctPtxP/lChz1gZQ7j3VGQWYipmAjEAh5FUTyiuF6j+/OLSsZhAmN2DYOYdP+30RhT2hX5cyUYdhaVgrYRuTBRyQtGQtd5K", + "id": "rs_edbdb8fa5c17554bb8c81cf0a739b84f", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative " + "values, and non-integer quantities.", + "type": "output_text", + }, + ], + "id": "msg_456a8eb8173957c694b6ef36746be4cf", + "phase": "commentary", "role": "assistant", "status": "completed", "type": "message", - } + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_5f196ff4cb915362abef090168884770", + "id": "fc_39bb6f400cd35d1fbad36e6cc2fdfaa0", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "in_memory", - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "safety_identifier": None, - "service_tier": "default", - "status": "completed", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 152, + "output_tokens_details": {"reasoning_tokens": 86}, + "total_tokens": 461, }, - "user": None, - "moderation": None, }, expected_litellm_response={ - "id": ANY, - "created_at": 1791240613, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_RESPONSE, + "created_at": 1792027275, "model": "bedrock_mantle/openai.gpt-6.1-sol", - "object": "response", "output": [ { - "id": "msg_b6510810920854d29a290136d5eb08b1", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "id": "rs_edbdb8fa5c17554bb8c81cf0a739b84f", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeGleCFHR3X7I/3TQIxhAYfW9mjX0cFKFSki7Gf5uzENSAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBMTVQNEFLb1hPRHNZMHUrbGhkYnExVzIveHBJMllVbENGQm5BY0RxMEVNZnh3c3BUS1NMV2t5N2o5SGRON2RuU1E9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcG493+ZcD13Vo9Z095jldqzOPdER/g3kHez3TwwTyexPQO0PepR8jRQ3zVYQVnB+5HsLoOUbfxtpDKFNZdhkG7vX7H79h6S2p5Tmcda0OnUhLEa1gj/FATh1bTrkCAAAQANAHetSqAaabgN4pn8RLag9s76ySEeodJgSjG7iLEOmNhVDDpO42M0mz4lH1lu3AXf////8AAAABAAAAAAAAAAAAAAABAAABw/bggpCep0rKr5bUUaXRidg+rbrNYiSeN0LEjiXKzD7Wt/+NhdY6fopjNz8a882sSCQGS6uQbFLsNO+b8+enVXmNgn96/OH9Nh89uQrTE/oPwYWD49Ts+5h4TJqljt1QVto3/i1qiVErPLn+0jpC8qZZKgOsUx8kApialKG9iJdnxT7xWsSSNALi5rt9ckiDVtw7HDLgzqIWs5d5kMBNj11466qX8pb+TYkQIAAFoR/Crx8KG3YqoX5kZJDZrUNFG0mz7arLDO9UaYG/ntFSqDDJmsYlcpyJchpzMZBNrO+5pQ4G8DIdBosJNnR/r5ebFJX7aLT0LWM5wI1morAlmNlhe+TOHPgY6AHpRuHu9cj1EWbuj1LKx+XQAu5NUEjIKFOFs+tVdENroClJnLtP1P4Ncn9DVjdy9pKzy6rt3UdpFkyIHizfPIHA9Ep6JEU37r2lzrJb2Bn9JtwRKxY7gB8TxUk/5tZ5DqzbPc+0KfFM9D1RL06U+birMIJwC8AM9zWpl0basljGg6Wg3t2Ey4Gua6M+gd5sGF0wQtDRShPH0VEy0dl6ZdfC8G0b1qxGWPpymastGVvw2CamaGJEuHexPr3//CytpLZYZmn/aIemUxH1AGcwZQIwF1zAfYPc+CW98SZN8eOtgH6Zvjg9iy37mmtat8ctPtxP/lChz1gZQ7j3VGQWYipmAjEAh5FUTyiuF6j+/OLSsZhAmN2DYOYdP+30RhT2hX5cyUYdhaVgrYRuTBRyQtGQtd5K", + }, + { + "id": "msg_456a8eb8173957c694b6ef36746be4cf", + "content": [ + { + "annotations": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative " + "values, and non-integer quantities.", + "type": "output_text", + "logprobs": [], + }, + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_5f196ff4cb915362abef090168884770", + "name": "read_file", + "type": "function_call", + "id": "fc_39bb6f400cd35d1fbad36e6cc2fdfaa0", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"summary": None, "effort": "medium", "context": "all_turns"}, - "status": "completed", + "reasoning": {"summary": None, "effort": "high", "context": "all_turns"}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -950,20 +967,566 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 152, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 86, "text_tokens": None}, + "total_tokens": 461, "cost": None, }, - "user": None, "store": True, "background": False, "billing": {"payer": "developer"}, - "completed_at": 1791240614, + "completed_at": 1792027277, "frequency_penalty": 0.0, "max_tool_calls": None, "presence_penalty": 0.0, "prompt_cache_key": None, + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, + "prompt_cache_retention": "in_memory", + "safety_identifier": None, + "service_tier": "default", + "top_logprobs": 0, + "moderation": None, + }, +) + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock_mantle/openai.gpt-6-luna"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-6-luna", + "input": BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_INPUT, + "instructions": RESPONSES_REQUEST["instructions"], + "tools": RESPONSES_REQUEST["tools"], + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027282, + "created_at": 1792027281, + "id": "resp_yc6hh2y7tm7sukz3zbg635xni33t6ozu2apke3askowhlad4lxga", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "responses client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", + "model": "openai.gpt-6-luna", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeOwVd15Aergyicha8AZ4SefU5Ab/+gNptGXd3dWnmXffAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBK2x2WjV3WXpBelNuNnBkWEtDeGMzeExnTkVVQXNndCtqckJRdC9EemJTWFdMNVpsOGFzSlFZWnd2MGFSUGxXMmc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBc+CaVDJzjT9x/Xl7UMI9TPO/qVRRjtLAYOLLWagTyexPQO0PepR8jRQ3zVYSYyTqscA+i/olxb+B4IMEyuMjfLB0apO50O6jWVoBau4lK1VAT7KOg4Ct3gcM8fpsCAAAQADY6nvfJNZKJekHDZgVZ6G5URFxJ5hQwwK85F4C79T8R+G6K6YQCqWfOAYynvUUXJ/////8AAAABAAAAAAAAAAAAAAABAAABjYWhLTlrtuV8NrFybrM5bSxs9syuqr0ZvNCbZuTrdEN+3NddWFCoXJfeEpKGsnNjgfg4LwxY//KIrX0MdnNRWVjq0Gm+ArDwWKMM46LGB8lWYeBkC+V+96UVABY5vw9CclMuLB/8Z0XHOqDAW/j8SEx9ag+3tRkt17OjZ/MbmwsrgIMJCr1Nnj3JsyKhPTvl07QGSj3BQIiLhpQAdXj/4PdVyRd2gcDq/kyQ9UJ/hlsxqTaZdlMAVyLCAecBxB0hBjsnQdknB/GkLtbSVxze/YAAMA2bvNuZ5O9NHSJk5R1JQKolDpSoyMKLhLm1LV0LDfoRdsbnXQfXhJpOaiTE/22X5fcwOoiRIcwzN+oQxYru9jMHDjWTMa8LjpcsH4etmfCwNS2TMQkAU6ZuQ1mteS8zsKZZjoBiqQKej/l5PC1VoLP8yjoMz9X5V1cyvj3I1CFS/tbtE34hn7LAfv1Go/VUhXnYHn2+vxgXr8kE78AW5NcmaNJ2duXCcako4LNqaWCa02joZsf8Z6C9IQqK8PAmLuDbPzaWLX/XNumnAGcwZQIxALG90gEcUh6QW0HMDRkZq2blkVDOUuObEpl+So92+Z8mPwLHC/k+yqnL4lccwlaLsQIwHxrXKIf4QCvwg9v9fStdw5+jU++ZmRIGjrtLoxRyLkV0NYjQ9y/cHwKh5grdA0Qr", + "id": "rs_23466129e3845ca49de5b72546c9370d", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, " + "and non-integer quantities.", + "type": "output_text", + }, + ], + "id": "msg_fb7c4c9f5fdc5280ae73896bbdb0b7bc", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":80}', + "call_id": "call_90b43b6346d45db185a8230ca05b7379", + "id": "fc_c4c1831a0bfe5c72b8ab072464830a14", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 125, + "output_tokens_details": {"reasoning_tokens": 60}, + "total_tokens": 434, + }, + }, + expected_litellm_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_RESPONSE, + "created_at": 1792027281, + "model": "bedrock_mantle/openai.gpt-6-luna", + "output": [ + { + "id": "rs_23466129e3845ca49de5b72546c9370d", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeOwVd15Aergyicha8AZ4SefU5Ab/+gNptGXd3dWnmXffAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBK2x2WjV3WXpBelNuNnBkWEtDeGMzeExnTkVVQXNndCtqckJRdC9EemJTWFdMNVpsOGFzSlFZWnd2MGFSUGxXMmc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBc+CaVDJzjT9x/Xl7UMI9TPO/qVRRjtLAYOLLWagTyexPQO0PepR8jRQ3zVYSYyTqscA+i/olxb+B4IMEyuMjfLB0apO50O6jWVoBau4lK1VAT7KOg4Ct3gcM8fpsCAAAQADY6nvfJNZKJekHDZgVZ6G5URFxJ5hQwwK85F4C79T8R+G6K6YQCqWfOAYynvUUXJ/////8AAAABAAAAAAAAAAAAAAABAAABjYWhLTlrtuV8NrFybrM5bSxs9syuqr0ZvNCbZuTrdEN+3NddWFCoXJfeEpKGsnNjgfg4LwxY//KIrX0MdnNRWVjq0Gm+ArDwWKMM46LGB8lWYeBkC+V+96UVABY5vw9CclMuLB/8Z0XHOqDAW/j8SEx9ag+3tRkt17OjZ/MbmwsrgIMJCr1Nnj3JsyKhPTvl07QGSj3BQIiLhpQAdXj/4PdVyRd2gcDq/kyQ9UJ/hlsxqTaZdlMAVyLCAecBxB0hBjsnQdknB/GkLtbSVxze/YAAMA2bvNuZ5O9NHSJk5R1JQKolDpSoyMKLhLm1LV0LDfoRdsbnXQfXhJpOaiTE/22X5fcwOoiRIcwzN+oQxYru9jMHDjWTMa8LjpcsH4etmfCwNS2TMQkAU6ZuQ1mteS8zsKZZjoBiqQKej/l5PC1VoLP8yjoMz9X5V1cyvj3I1CFS/tbtE34hn7LAfv1Go/VUhXnYHn2+vxgXr8kE78AW5NcmaNJ2duXCcako4LNqaWCa02joZsf8Z6C9IQqK8PAmLuDbPzaWLX/XNumnAGcwZQIxALG90gEcUh6QW0HMDRkZq2blkVDOUuObEpl+So92+Z8mPwLHC/k+yqnL4lccwlaLsQIwHxrXKIf4QCvwg9v9fStdw5+jU++ZmRIGjrtLoxRyLkV0NYjQ9y/cHwKh5grdA0Qr", + }, + { + "id": "msg_fb7c4c9f5fdc5280ae73896bbdb0b7bc", + "content": [ + { + "annotations": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, " + "and non-integer quantities.", + "type": "output_text", + "logprobs": [], + }, + ], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":80}', + "call_id": "call_90b43b6346d45db185a8230ca05b7379", + "name": "read_file", + "type": "function_call", + "id": "fc_c4c1831a0bfe5c72b8ab072464830a14", + "namespace": None, + "status": "completed", + }, + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], + "top_p": 0.98, + "reasoning": {"summary": None, "effort": "high", "context": "all_turns"}, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 309, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 125, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 60, "text_tokens": None}, + "total_tokens": 434, + "cost": None, + }, + "store": True, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1792027282, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, + "prompt_cache_retention": "in_memory", + "safety_identifier": None, + "service_tier": "default", + "top_logprobs": 0, + "moderation": None, + }, +) + +GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock_mantle/openai.gpt-6-astra"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-6-astra", + "input": BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_INPUT, + "instructions": RESPONSES_REQUEST["instructions"], + "tools": RESPONSES_REQUEST["tools"], + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027291, + "created_at": 1792027288, + "id": "resp_zq4paxdqatt7bzmozrzd5dgkc45z2fqq2t62p65iytdvzhpj4imq", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "responses client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", + "model": "openai.gpt-6-astra", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeGJfCWQXacPcv8CJEsM/9LMzYWLNLvINpMJiB+7mMoMSAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBdmV6d1pveEZZenhoM3ZWZUxEOTBQZ09rcjBXYys0eEFIdmtnMVhZajdPT3J0MHpFVWJvLzZuTjJGd0lMOEhzWmc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcqmn6iHlcPI0BGSnwJYE1d9MzPK9bnaHOZk9lFQTyexPQO0PepR8jRQ3zVYTqc2mAtpep3MH2qkAqIiac9vmFBptiTdAS408lL863oq6JCKa922ci0EO0NO/j7q8CAAAQAO0SFcb1ccaIX77cujd+ERFaa3QyF85pu8HuGJKKkp6S3ANY44npxMkRFkXKvlyLtv////8AAAABAAAAAAAAAAAAAAABAAABszzAzyhxSMCZO+FKUTOV9cQK8fNZeqnOi2vw5VedZ2dLeEc7gJtXDVgTNL2lBRcMwhPCeSy5z0W3086Lp4JsUt2rsVSGsrmTA1ZwOrPRGTn9D2blXhSKrNkAku9omlqm5nUj9g/eqxKNL8xl+J5FAcIGDJe2KJAXrgA63Zg5Ry5y+VlyNGrsQec5gyGL3m8Fqiz1dOfUJ7kPyakAFz372sEeqCX01OjcKj8W+gdD63sSX4rmnO9aVne2M1N2EjqT/GehHTWBue/7lSnjg8cdg76Y5dGNYdY8EJhuKlp7ARqiJKUpPcA6qOzusPSiF6h5dbYbhp89HtC10iTP4DSnL1lMMpT4WulwcScshvOVFGFAHj7PtwwPR2EAvSwfvmJEYbAfeyEMJL+WY1LdZS9+dFkb0uWo8C3KnivTWsORS0vYmGcgpK2AVTknF0it0rV7AQ2kJ1XuVbYVMAF8mjc0dFURDwewPFCxbclWCtyHFCpIEQQwuLtCjPTeaYzBVyx55JsLn2Ubdbj/afbjUihNCVh7HJynKS4y1a76YJ9Ju6Vj2Yo1qbkSxsVouwCXpm1HyWR9tFYrHiONBNxA8fqSgGoYKLcAZzBlAjEAwcTlQMfxQqW70KnwfplmXOQi/CeaoAf6gCkfOLmJKfA3o5ZXGaypcmFB8kx3BS4YAjBp573CdX9BLzVIQv15fXhlL75Txr46SCK/kkYmvQxpGawvajLjTQR8EaJlJQXuNdw=", + "id": "rs_81e1dfdd19dc5140b879747fddfcc5a4", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, " + "and non-integer quantities.", + "type": "output_text", + }, + ], + "id": "msg_9be7ef044f5159d7812e99131a4d75cd", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_38b08e80239e571e8f918025be1a128c", + "id": "fc_13ce596fb38a5b50a123a6e2357705bf", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 143, + "output_tokens_details": {"reasoning_tokens": 78}, + "total_tokens": 452, + }, + }, + expected_litellm_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_RESPONSE, + "created_at": 1792027288, + "model": "bedrock_mantle/openai.gpt-6-astra", + "output": [ + { + "id": "rs_81e1dfdd19dc5140b879747fddfcc5a4", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeGJfCWQXacPcv8CJEsM/9LMzYWLNLvINpMJiB+7mMoMSAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBdmV6d1pveEZZenhoM3ZWZUxEOTBQZ09rcjBXYys0eEFIdmtnMVhZajdPT3J0MHpFVWJvLzZuTjJGd0lMOEhzWmc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcqmn6iHlcPI0BGSnwJYE1d9MzPK9bnaHOZk9lFQTyexPQO0PepR8jRQ3zVYTqc2mAtpep3MH2qkAqIiac9vmFBptiTdAS408lL863oq6JCKa922ci0EO0NO/j7q8CAAAQAO0SFcb1ccaIX77cujd+ERFaa3QyF85pu8HuGJKKkp6S3ANY44npxMkRFkXKvlyLtv////8AAAABAAAAAAAAAAAAAAABAAABszzAzyhxSMCZO+FKUTOV9cQK8fNZeqnOi2vw5VedZ2dLeEc7gJtXDVgTNL2lBRcMwhPCeSy5z0W3086Lp4JsUt2rsVSGsrmTA1ZwOrPRGTn9D2blXhSKrNkAku9omlqm5nUj9g/eqxKNL8xl+J5FAcIGDJe2KJAXrgA63Zg5Ry5y+VlyNGrsQec5gyGL3m8Fqiz1dOfUJ7kPyakAFz372sEeqCX01OjcKj8W+gdD63sSX4rmnO9aVne2M1N2EjqT/GehHTWBue/7lSnjg8cdg76Y5dGNYdY8EJhuKlp7ARqiJKUpPcA6qOzusPSiF6h5dbYbhp89HtC10iTP4DSnL1lMMpT4WulwcScshvOVFGFAHj7PtwwPR2EAvSwfvmJEYbAfeyEMJL+WY1LdZS9+dFkb0uWo8C3KnivTWsORS0vYmGcgpK2AVTknF0it0rV7AQ2kJ1XuVbYVMAF8mjc0dFURDwewPFCxbclWCtyHFCpIEQQwuLtCjPTeaYzBVyx55JsLn2Ubdbj/afbjUihNCVh7HJynKS4y1a76YJ9Ju6Vj2Yo1qbkSxsVouwCXpm1HyWR9tFYrHiONBNxA8fqSgGoYKLcAZzBlAjEAwcTlQMfxQqW70KnwfplmXOQi/CeaoAf6gCkfOLmJKfA3o5ZXGaypcmFB8kx3BS4YAjBp573CdX9BLzVIQv15fXhlL75Txr46SCK/kkYmvQxpGawvajLjTQR8EaJlJQXuNdw=", + }, + { + "id": "msg_9be7ef044f5159d7812e99131a4d75cd", + "content": [ + { + "annotations": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, " + "and non-integer quantities.", + "type": "output_text", + "logprobs": [], + }, + ], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_38b08e80239e571e8f918025be1a128c", + "name": "read_file", + "type": "function_call", + "id": "fc_13ce596fb38a5b50a123a6e2357705bf", + "namespace": None, + "status": "completed", + }, + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], + "top_p": 0.98, + "reasoning": {"summary": None, "effort": "high", "context": "all_turns"}, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 309, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 143, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 78, "text_tokens": None}, + "total_tokens": 452, + "cost": None, + }, + "store": True, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1792027291, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, + "prompt_cache_retention": "in_memory", + "safety_identifier": None, + "service_tier": "default", + "top_logprobs": 0, + "moderation": None, + }, +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "bedrock_mantle/openai.gpt-5.6-terra"}, + expected_provider_endpoint="/openai/v1/responses", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-mantle-key", + "content-type": "application/json", + }, + expected_provider_request={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST, + "model": "openai.gpt-5.6-terra", + "input": BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_INPUT, + "instructions": RESPONSES_REQUEST["instructions"], + "tools": RESPONSES_REQUEST["tools"], + }, + mock_provider_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE, + "completed_at": 1792027296, + "created_at": 1792027295, + "id": "resp_2b4bqitdtukdshe6dbwxc72wu4tzjgbgo5nqe2azpevkccamxwha", + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "responses client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", + "model": "openai.gpt-5.6-terra", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJpX16vMhsuUqP/uVtfuWta1E1ddkEa7eTm4pRCs6dffAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBejQ3QS9nd0FYVXlSWk9EUm9aUkxOWDRCSkE5QUFWdmFOeGFUVmlnUkVadEtpb0lEVkRlZ0NET2QwcGVvRk1ZOFE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBckDtuTh3Rf5+ICZFJ9KAZsgYDTlLg1gBVDY7YvwTyexPQO0PepR8jRQ3zVYQIZ632aDn1RndzwIXGL3BxgsZyacOxdIuUi/jAwMN8xj+g0LUR7IpBpxhhW/PUcJ4CAAAQAFs2qematU6KvhLcfJq262dV/lMVbdgmfmRH+Hv8Fw4Im91UxrkD9vqufYM0I3k1XP////8AAAABAAAAAAAAAAAAAAABAAABBi+t1YoVoMFOwxnA4vZEsOt2Rptfcl+xMaPmGZuFL6MXusOn82/F/A/5JCEQrjwGL8zpRZUzjc24LEufSMVhgxJ6sRld+5yP2siwkdWMitQYC0Ocazo5ZYh4uwM+m7JHT6WBIzyDXAEWql/fPiXL/CNl77RgapLBVBd8PyqpALtZApt3fmfE8BRTLWZsu1+rouEJTicpE0mtgOYdRl0wyEhYYLpV4XSk/oBxNDdfL6GnOISnSK3MtYQ9mgoIHjKjfkjsCEILvCM2R1GiFb2Z6j13LjdeAbAG6Ad1WrjXB9V9opjCMxzffOL0VxSqrUrurbYJ7IJdvoRmlnuHixJ4cEXPbUyhIFbGVx7JxBXvLtUP9YemsZI2AGcwZQIwKBicwyYY0SSw2IVndhjAzPEudeT0mtAuI4cNZYLocLdsAmZ0CMDtqzb9L/ZgqYYuAjEAgiwJ9MrLB1A/3s9OjxkrAPWFdwARdvuq8yfJiV7hhhAiFimDIKDx1NPywAUZcoRg", + "id": "rs_63551860cd6051cebdb9f7ede8656968", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I need `shop/validation.py` next to inspect the current handling of zero, " + "negative, and non-integer quantities.", + "type": "output_text", + }, + ], + "id": "msg_3b2f55f61ac45802b1344d0759287853", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_fdd25b3193de5afb9ffba7c630ae90f2", + "id": "fc_db3ddf54911b51569ac9c6a05f31385a", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 81, + "output_tokens_details": {"reasoning_tokens": 16}, + "total_tokens": 390, + }, + }, + expected_litellm_response={ + **BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSES_RESPONSE, + "created_at": 1792027295, + "model": "bedrock_mantle/openai.gpt-5.6-terra", + "output": [ + { + "id": "rs_63551860cd6051cebdb9f7ede8656968", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJpX16vMhsuUqP/uVtfuWta1E1ddkEa7eTm4pRCs6dffAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBejQ3QS9nd0FYVXlSWk9EUm9aUkxOWDRCSkE5QUFWdmFOeGFUVmlnUkVadEtpb0lEVkRlZ0NET2QwcGVvRk1ZOFE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBckDtuTh3Rf5+ICZFJ9KAZsgYDTlLg1gBVDY7YvwTyexPQO0PepR8jRQ3zVYQIZ632aDn1RndzwIXGL3BxgsZyacOxdIuUi/jAwMN8xj+g0LUR7IpBpxhhW/PUcJ4CAAAQAFs2qematU6KvhLcfJq262dV/lMVbdgmfmRH+Hv8Fw4Im91UxrkD9vqufYM0I3k1XP////8AAAABAAAAAAAAAAAAAAABAAABBi+t1YoVoMFOwxnA4vZEsOt2Rptfcl+xMaPmGZuFL6MXusOn82/F/A/5JCEQrjwGL8zpRZUzjc24LEufSMVhgxJ6sRld+5yP2siwkdWMitQYC0Ocazo5ZYh4uwM+m7JHT6WBIzyDXAEWql/fPiXL/CNl77RgapLBVBd8PyqpALtZApt3fmfE8BRTLWZsu1+rouEJTicpE0mtgOYdRl0wyEhYYLpV4XSk/oBxNDdfL6GnOISnSK3MtYQ9mgoIHjKjfkjsCEILvCM2R1GiFb2Z6j13LjdeAbAG6Ad1WrjXB9V9opjCMxzffOL0VxSqrUrurbYJ7IJdvoRmlnuHixJ4cEXPbUyhIFbGVx7JxBXvLtUP9YemsZI2AGcwZQIwKBicwyYY0SSw2IVndhjAzPEudeT0mtAuI4cNZYLocLdsAmZ0CMDtqzb9L/ZgqYYuAjEAgiwJ9MrLB1A/3s9OjxkrAPWFdwARdvuq8yfJiV7hhhAiFimDIKDx1NPywAUZcoRg", + }, + { + "id": "msg_3b2f55f61ac45802b1344d0759287853", + "content": [ + { + "annotations": [], + "text": "I need `shop/validation.py` next to inspect the current handling of zero, " + "negative, and non-integer quantities.", + "type": "output_text", + "logprobs": [], + }, + ], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_fdd25b3193de5afb9ffba7c630ae90f2", + "name": "read_file", + "type": "function_call", + "id": "fc_db3ddf54911b51569ac9c6a05f31385a", + "namespace": None, + "status": "completed", + }, + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], + "top_p": 0.98, + "reasoning": {"summary": None, "effort": "high", "context": "all_turns"}, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 309, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 81, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 16, "text_tokens": None}, + "total_tokens": 390, + "cost": None, + }, + "store": True, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1792027296, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, "prompt_cache_retention": "in_memory", "safety_identifier": None, "service_tier": "default", diff --git a/tests/integration/translation/responses/bases/gemini.py b/tests/integration/translation/responses/bases/gemini.py index 34591a31843..597dbc25ab7 100644 --- a/tests/integration/translation/responses/bases/gemini.py +++ b/tests/integration/translation/responses/bases/gemini.py @@ -1,7 +1,501 @@ +from collections.abc import Mapping from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import RESPONSES_REQUEST, RESPONSES_RESPONSE +from integration.translation.templates.gemini import GEMINI_GENERATE_CONTENT_REQUEST, GEMINI_GENERATE_CONTENT_RESPONSE +from pydantic import JsonValue + +GEMINI_RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in RESPONSES_RESPONSE.items() if key != "provider_specific_fields" +} + +GEMINI_3_8_FLASH_THOUGHT: Final = ( + "**My Approach to Analyzing Quantity Validation**\n\n" + "Okay, the user wants me to understand how zero and negative quantities are handled, and they’ve " + "given me a specific constraint: explain my next move and then call a read-only tool. I need to b" + "e careful to avoid making any changes. The overall goal is to review the project, so it's a good" + " approach to use the responses client to keep track of the process. I also need to be mindful of" + " zero, negative, and non-integer quantities in my analysis.\n\n" + 'The user specifically mentions "reading the validation file first." Let\'s start with `shop/valid' + "ation.py`. I need to figure out how they are checked in the `validate_quantity` function, so it " + "makes sense to examine that file. But before diving too deep into all lines, I should try to min" + "imize how much data I read. Given that the tool's performance might become degraded when reading" + " large files, I should consider reading only a portion of the file. To get an overview, I will r" + "ead the first 50 lines to see if that's enough information.\n\n" + "Following the instructions, I need to clearly state my next step before executing it. Therefore," + " I'll explain that I need to examine `shop/validation.py` to understand how the system handles z" + "ero, negative, and non-integer quantities, focusing specifically on the `validate_quantity` func" + 'tion, then I\'ll use `read_file(path="shop/validation.py", start_line=1, end_line=50)` to read th' + "e start of the file.\n\n\n" +) + +GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = ( + "Er8KCrwKAWkUfROyraCcwrIYpIws/Jh5cMTrVJ2ShZLiHvXJuIq/KdxErXHBEeWq4H9FXkFOU5eZFSvo6lnkwhw09W+nukH2" + "Jgcjan4KoNsFIGvNKhF/mT66xCr40t/LHRSIxu4o585L29auyheBJmh3cR4NjnZPYwLKzmnI3W2ePhcD5tjLU/pt6LOX4IBN" + "jsDho8ep9wW2foJDkEorvAYRI8RiXJ2loNEjagJAaSo2Rt3URPz3KzvwKo8QlJAYGiQ+5eUkVnwcCsNs0NNY5+RVggqnJflK" + "okqjOsaIXAxo2jupI4amtI+NpQ59K5TddnJ8eh+fiQSphB2Ex2AlhczWhoXbYqdkjRfYtXF1KiTO4w4L74ApYd7iMSm1Ix5U" + "76yYiB5+sRaZYzuBIdD01O+SMqPXil9Ft73kzuH6CROVi5Xf3RMRAQvmuJcRYciprH0kcba7E2Z6c0v7Ht6kegzoWrpAtGgW" + "hL2KTMewyWQF7IZmYLGoo20hniIliBs+7bLAPJ3DEsp1HU55dAsdvV1IwQZtFuqKffE7MB0uQws/snCf6w9mpCbVOYRVuzZe" + "LTbWzar84bmsp1dfHGS2gOkvsi5fi8IUs/be9aK/budI6nkSXborMoESSG3Z5BTXgi5U0vJcS2FTO6oFMpE1lPBL/vIpPDKR" + "ujjqtUT+QA89SJmLZDORPc3aLd36dEErhlSZ+9LYsU3EkKNruomCdiSrbClIRK50YYu/ocG9lXF4yhUp3dgKbMWCw9jGZv+a" + "KPz8sXVgbXSm+zLuvtByXHSfiVmu7hP1Oc7BmdL+lJW03rgMAxtDb39rbP8l+5sm4EDJil9mcGGOYpcBSKRmDacoqk9D8AWG" + "PwlYJ6L1EfsxKXQNFXa9l4kEv4QXn2d5BQnfCjFwsnolQmeSWP2XYh8O+R/YmhhFIAZhvzMctHnQ/XRRMnjADlpnM1mioyy4" + "poDsUDKIyr92JvDK+T+1NRU/uQva9ScCF9YJCM3uLdxkocwx8eBN0hsOZYosrUq9FLL0+aNgHgybUyn4Kef197W4fht3yS2Z" + "AUuNm4nM3VXRKlLgP3MYJnkOlBTVooyEoqC6raGH2RMdIegAs5JSIyd6xHkg8J2DaLhy+7WbpwqcHHCnM67sVyfRqQC9TP1a" + "D0C0saoNL8sAZfHiDQPOLmvxt9z2+QpYeNZprWgGIGw5iZmLPexGHNLFd+i063HUpf5dYmjPwdL7970CW8PPQgjxW7OMOTlm" + "Gkbf4YMsyiTrHbmVBjVPm3dA93q7wE3zsuhJ2icHkT7zlvHzz+nYtgJlWyoMtOPOjUunNM2pVXmMtXou0Rp0aR7YDIy9uqnY" + "GIfBKYriMp/wL1apCyzih0EqcGhqNVspDDPJV0Slj3VKP7LaBTqz0LdTFo1WZAZNUZJW2sOHC8c9oXjNCM/o25qOPHowmzN9" + "kWFBAtuetPHn5lD5PFvIdYMyjOtwRpFsPyAM8E6Nb7hP3pQnW8zvQ9YrfT6nqst39f5JHyjXh0HtVJgF+EjETS8H2EJSY3a9" + "ExXPCRmCyXnKR9haJ95o+XRVMlJRucw9C34RwZ/po7oivV5jV+hSSOnTZuxx7rGH6md0ELhMKXO3PwaNE/s14SqOCBc9s9yj" + "vGbC9fVe89KWNigVeejbIVlSs8uoHld9Bhespx8zZVheVQP9Oc9lVLXzmLteo4GNKkFYoJ7a5kCvx85vj9vAqKISOBuc6Vfu" + "zH3cl8BYRVo5JiaxWRQxcTAtTbhH/RxOXHm+gc8gtdisilWLHaS13D0gmaq28gkUTCg=" +) + +GEMINI_3_8_FLASH_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_8_FLASH_THOUGHT, + "thought": True, + }, + { + "text": ( + "I need to inspect `shop/validation.py` next to see how `validate_quantity` handles zero, negativ" + "e, and non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "start_line": 1, + "end_line": 50, + }, + "id": "call_23592", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 64, + "totalTokenCount": 750, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 332, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "-CbQavzyBouHz7IP-Ma2uQw", +} + +GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "gemini/gemini-3.8-flash"}, + expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request={ + **GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": {"parts": [{"text": RESPONSES_REQUEST["instructions"]}]}, + }, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_3_8_FLASH_RESPONSE_FIELDS}, + expected_litellm_response={ + **GEMINI_RESPONSES_RESPONSE, + "model": "gemini/gemini-3.8-flash", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": GEMINI_3_8_FLASH_THOUGHT, + "annotations": [], + }, + ], + "phase": None, + }, + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": ( + "I need to inspect `shop/validation.py` next to see how `validate_quantity` handles zero, negativ" + "e, and non-integer quantities." + ), + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "call_id": "call_23592__thought__" + GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + "name": "read_file", + "type": "function_call", + "id": "fc_call_23592__thought__" + GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + "namespace": None, + "status": "completed", + "provider_specific_fields": { + "thought_signature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + }, + }, + ], + "usage": { + "input_tokens": 354, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 354, + "video_tokens": None, + }, + "output_tokens": 396, + "output_tokens_details": { + "audio_tokens": None, + "reasoning_tokens": 332, + "text_tokens": 64, + }, + "total_tokens": 750, + "cost": None, + }, + }, +) + +GEMINI_3_1_PRO_PREVIEW_THOUGHT: Final = ( + "Okay, here's my summary, taking on the persona you requested:\n\n" + "**Navigating the Validation Landscape**\n\n" + "Alright, before I dive into the core logic, I really need to understand how the system handles t" + "he edge cases – specifically, zero and negative quantities. I've got a gut feeling those could b" + "e potential vulnerabilities or, at the very least, a source of unexpected behavior. So, my initi" + "al move is clear: I'm going to dig into `shop/validation.py`. Understanding the validation strat" + "egy upfront is critical; it informs everything downstream. This will give me a clear picture of " + "how the system is designed to handle invalid or potentially malicious inputs and will set the st" + "age for a more focused review of the main logic. I'll read that file first.\n\n\n" +) + +GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = ( + "Ev8BCvwBAWkUfRM62a1j2olCssVKs9GECDQ5L1C7yrRsGmPVGbcIm+MrafD721G6/xzw4hoxWRaLYt24NwcS8gLJMwZMKF2U" + "xOKF2qPxQIrxVkY0JG8s9A1vVBlu4mNsDGoBMPLc3CEX/Wb7sJ8u/oaXz8n/4yWDc9NgLwxdpzrDH9An0fmu2MGpZ+rL1nHj" + "E+ZoNp6gKpaiRe3MmdBF2+Oy3MI6JZQnU9C/T9InT2F6NyzdZMhqSXhTf5qddxt9XLeabg8TKO4+GLKlmjB/T9NBOP00fQqS" + "rLtPHkoQgxhn3S0Qj5kDAOgNXvJM4lyhlGyWIfBuHNN3VI9LpfVMhKU2" +) + +GEMINI_3_1_PRO_PREVIEW_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + "thought": True, + }, + { + "text": "I will read the validation code first.", + }, + { + "functionCall": { + "name": "read_file", + "args": { + "end_line": 50, + "start_line": 1, + "path": "shop/validation.py", + }, + "id": "call_25410", + }, + "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 354, + "candidatesTokenCount": 41, + "totalTokenCount": 429, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 354, + }, + ], + "thoughtsTokenCount": 34, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "_ibQasKwGeCBqtsP84vwkAk", +} + +GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "gemini/gemini-3.1-pro-preview"}, + expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request={ + **GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": {"parts": [{"text": RESPONSES_REQUEST["instructions"]}]}, + }, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_3_1_PRO_PREVIEW_RESPONSE_FIELDS}, + expected_litellm_response={ + **GEMINI_RESPONSES_RESPONSE, + "model": "gemini/gemini-3.1-pro-preview", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + "annotations": [], + }, + ], + "phase": None, + }, + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I will read the validation code first.", + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"end_line": 50, "start_line": 1, "path": "shop/validation.py"}', + "call_id": "call_25410__thought__" + GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + "name": "read_file", + "type": "function_call", + "id": "fc_call_25410__thought__" + GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + "namespace": None, + "status": "completed", + "provider_specific_fields": { + "thought_signature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE, + }, + }, + ], + "usage": { + "input_tokens": 354, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 354, + "video_tokens": None, + }, + "output_tokens": 75, + "output_tokens_details": { + "audio_tokens": None, + "reasoning_tokens": 34, + "text_tokens": 41, + }, + "total_tokens": 429, + "cost": None, + }, + }, +) + +GEMINI_4_ARGON_THOUGHT: Final = ( + "Okay, here's my thought process on this task:\n\n**My Approach to Quantity Validation**\n\n" + "Alright, the instructions are clear: I need to understand how this system deals with potentially" + " problematic quantities like zero, negative numbers, and non-integers. The prompt explicitly dir" + "ects me to start with `shop/validation.py`. This makes perfect sense; it's the logical starting " + "point for understanding the system's core logic regarding quantity checks. I also need to rememb" + "er the system instructions regarding the use of the `read_file` tool and not making file changes" + ".\n\nSo, I will state clearly that my next step is to examine `shop/validation.py` to understand i" + "ts handling of these edge cases. I'll then use the `read_file` tool on that specific file. This " + "will provide the necessary context before I can even *consider* any potential fixes or further f" + "ile inspections. This approach aligns with the instructions and ensures I'm building a solid fou" + "ndation of understanding before I start looking for specific issues.\n\n\n" +) + +GEMINI_4_ARGON_THOUGHT_SIGNATURE: Final = ( + "EsIFCr8FAWkUfROZ8zCh3vM4ufC+mcdQi7BamG510F5TkuWeQp8lVro8DkxrDl/ZyWgcPR4JVCGdPY4copLFyp/2Qz4zjr2P" + "oQxUmWbFtBm66uN4aYxLhcGXwwB3BIWXezLRJXaVoJl6ex/VrXnos2M/3+CHVKM3ThtPYYsZjpeqJgnHsYYZi3X/sl+G4214" + "OL2X+pSGsxESSoj4EFOqHb774e9smvmLzNbca2cEDuo5dvoGlkdgALSL2ODIVi/625uid3Sbf+YAhj8aIoD4/e0LDgbRYilG" + "o5MsGgLFQfyEjjFc0N0NxL02T+9qjlagPHSe74VAsWydKGgFQhgk9MOy9bIdJ+9RE1CWYjAaHXwagQenSa9tLq1bCNo5RXJF" + "2aRrT+pWc8KwUL8HaqYEytvT9yHyTm99nMDFTMPpCDcsjIKmlG0FABQfEpBT9cCjZMsRR5hfK2voTRcjFvGecHBD71XX3Q+N" + "0p12WK87Z9Hw8OQgWIYdamOtMDw2n24m1fhC1fEJbSzOC3PzDwZOAUPemFxHpi9Vo1TnONRBX4wEAWhdigP5qr/CSmUPiiFm" + "c7Ay8rxezmoqQ8bs1zBYnFzAhBMAHfc+lZ2uEmoaIK1b0FydfDTZXpeF+ayxLz9OPhU4jrKJ3n5tywrQvx7ruwhIlJpX1sfx" + "OfyRbrNbOkX0ruTvyvGRI34kIQUoMK4M6oAUo8n+W3DGu7AO+lljNkVanLaAlJwCc/9r4E8GMKpRnqpit+ia2NUDi+U4/Ob3" + "XOxODNUzd2DgAmgXmcHczCXa/SSfy/wFqW21QYXzbPgpAboF/qVV17q7NU1DCXtPamowkMERCPB6Z5FZI34dPToDXyYQGxak" + "MVAH6f0vZUrpakrs6uXFR5RdwOEUzcPfQq7nwskzrvWCzyNk8YnXJjccpGCF/cXRQW+MKV17bj9xjWZguw==" +) + +GEMINI_4_ARGON_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_4_ARGON_THOUGHT, + "thought": True, + }, + { + "text": ( + "I will read `shop/validation.py` next to see how zero, negative, and non-integer quantities are " + "validated." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "start_line": 1, + "end_line": 50, + }, + "id": "call_31641", + }, + "thoughtSignature": GEMINI_4_ARGON_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 412, + "candidatesTokenCount": 65, + "totalTokenCount": 614, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 412, + }, + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65, + }, + ], + "thoughtsTokenCount": 137, + }, + "modelVersion": "gemini-4-argon", + "responseId": "AyfQauWvIqOO6dkPs7f0GQ", +} + +GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "gemini/gemini-4-argon"}, + expected_provider_endpoint="/models/gemini-4-argon:generateContent", + expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, + expected_provider_request={ + **GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": {"parts": [{"text": RESPONSES_REQUEST["instructions"]}]}, + }, + mock_provider_response={**GEMINI_GENERATE_CONTENT_RESPONSE, **GEMINI_4_ARGON_RESPONSE_FIELDS}, + expected_litellm_response={ + **GEMINI_RESPONSES_RESPONSE, + "model": "gemini/gemini-4-argon", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": GEMINI_4_ARGON_THOUGHT, + "annotations": [], + }, + ], + "phase": None, + }, + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": ( + "I will read `shop/validation.py` next to see how zero, negative, and non-integer quantities are " + "validated." + ), + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 50}', + "call_id": "call_31641__thought__" + GEMINI_4_ARGON_THOUGHT_SIGNATURE, + "name": "read_file", + "type": "function_call", + "id": "fc_call_31641__thought__" + GEMINI_4_ARGON_THOUGHT_SIGNATURE, + "namespace": None, + "status": "completed", + "provider_specific_fields": { + "thought_signature": GEMINI_4_ARGON_THOUGHT_SIGNATURE, + }, + }, + ], + "usage": { + "input_tokens": 412, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 412, + "video_tokens": None, + }, + "output_tokens": 202, + "output_tokens_details": { + "audio_tokens": None, + "reasoning_tokens": 137, + "text_tokens": 65, + }, + "total_tokens": 614, + "cost": None, + }, + }, +) GEMINI_3_5_FLASH_THOUGHT_SIGNATURE: Final = "ErkECrYEAWkUfRORlQ5j/gkLmyx08FVeIMgg57FJXrjcLwBlt9iuELI6Uc5h+NR/Vm+gEsbdZ5mgFTmKjPF9K/eft8IlXMKm6odJjQOgOYsbt2JHzANo0bpfTmlA6fIi0G2zLbvBVASA6Bdxu1aPQuO4voioQRwm2vomRxH1YbWp8sKXk0DBVbefosldrL0zLJFsi5dFYCtPvw0n9olPVgptHzEdiyXqG+63aPxTooARRQutUH0XWAKR0V+P7qWPt55QKlLaKQqKeBndg9JrEplJihg1sp++y+NAi18fsqXteUS2zIeDtdePGM/GS5oVibwE25zJoziPRdtJhGasFSaA7a3znhW9PF0pBIPAIRPKE4NsQ0FRhpy7ksIXY+0uJ4N+WPPejrtKK6z5x+P0tFkFP0ZNNPM8FZbir1ncVhVxkZS/wWmhc/8TZoRA9ghlTpYhHJ+C4fRVqQqnyRR3SDpVTzB4/sCjBlb434dTH0U3jB4h6V9b/Zx4k4pwUwZTNr2FfgOt2bR7u05DOa+H73OzsNG6zBnMYgBndQdRgk58+l4+UcZdpGKB0lkbHdfD2bminBypEmeJKNRpuc7Smuu0YxcZiY03tzzhHdrUmItqC39OEr2CzRcT9DjFpiWydo3ej9ZkEXeyxoMCckpMmGWh6xbjAnX9gkrPFmUE2rJqblDJWa51i6u/p9Y6ciCq6j4lAy9eBfULIRGQt9pKOXagltOnX0vR0MTDYgWe4dVDSzCI1TV7X/SQYjo=" @@ -93,185 +587,3 @@ GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( "store": None, }, ) - -GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = "EtsDCtgDAWkUfRODYoLRWC+DajYQxOvsLgPh0m8j4NTnd7BflzgPBKFfPW+PU1XsMQuEzviI1qk5mYI0qCfOQNf84PAXXvFA5hMYl+YObaND4G+ZtCdYcolFVfPJQqgK6Kpv20n9hZfLt5JzOS2+HRCLZaokIsZFadN++wqEeEkWQhnKdLGH1lM0fn8Fj/pYq95YLGnB90B8Oaj4qyG6ost2dzRAeAzFSXAko1mD/IgsDrDhEumngCqotdAbPW4jUGYOGDpoXLrBzQZvGa9blRC3ep6NLT0EYMnXImLFoZaLLIBMzVDsmmL0qOg4Gu+uNJlY6cDmtqRgkrcvuvGhh8+lrjUMJVigSsTAoKsTnT3OyCqdNqa+R2aD4WTl1uBFyGY7yXpZ9skQPkV210QNOylZ6exaMA51+W/mohL5j5+OJX7xtVfRIpjp4e0PLkEPxnuviX6OU4ykWZSSiztSXzogbrmnwP7faclRXTXHE5pFlM8y7gDJ9NEwxd5vvIMLdaK2dRWBNtvjI3Cj56sLKygH/j0mQSnt3PonE6Jv6XMJDPbuIzsjzX8t/j3u6WP0bVGnifNNEfIqt6YcKgeWlUTnabRGkWg/fBSyYbiTTEIGhketv57hA9oH7jJZbw==" - -GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "gemini/gemini-3.8-flash", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, - expected_provider_request={ - "contents": [{"parts": [{"text": "Say hello."}], "role": "user"}], - "generationConfig": {"max_output_tokens": 1024}, - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], - "role": "model", - }, - "finishReason": "STOP", - "index": 0, - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 2, - "totalTokenCount": 120, - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10}], - "thoughtsTokenCount": 108, - "serviceTier": "standard", - }, - "modelVersion": "gemini-3.8-flash", - "responseId": "5BjEaunaCtadz7IP-8WKsAk", - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "gemini/gemini-3.8-flash", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 10, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 10, - "video_tokens": None, - }, - "output_tokens": 110, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 108, "text_tokens": 2}, - "total_tokens": 120, - "cost": None, - }, - "user": None, - "store": None, - }, -) - -GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = "Ev8DCvwDAWkUfRMrwgm49bezSdfm90OeW6KeR4kKqExF8s+EorBlW6NWc6XLqdQ2dA5xZ4p0CPNrjuxemR5dt9ch3qMTSgIHTcKKAnxJ0eqsjR61EJWFP4JNdigiymymc7UNs/zLZer+qPH2XQLD9r85O3NVBeupYk6xy6395CZYygF9oVJD3WNXwlefvThnvH/3rDsnO0FBfcrvxRHiSVTD1Moe+uTVV2w3vKKSCxUb64w5lquEjFx+AO/jiJIc3McPvAOvUr0I/2fMCWLcO7Y5sV6zuN8qpaQKitC/Ev09cl3SAbKzwjO3gBgYVmF5PcPY5HT8S3bwCez2aOgX7BCN+FItlZ4wMsZStLIY38XMQtbibVRHmiufN86IMkoD8Yxb1lnK7aaS+anSPkn4M4zkkiwAUdjt14k6JvMBk9J4duvTHLP0BiSaLLkTe6Ufj9cRMxZKCP0ew4DiLMwejuQoSpC4aLP1gmli4eOUcyq/g2/o0kByY8Fl2vb54eiXADp4fhhIEAZFe4J/0x29ZZVKx8KEInDahP7tslzwah3PWfX/K1jXzqX93mo4a/0Ec1bM/sqwiEWSHNhrPpyXcTkmWQ1Bot+PTARnzAjeXvoTR5sajbNKj1SdCC0RY7zb8OqGZHDQxgNw7ghTjYMTdBJsdCMQfnl3dvqmw6zM/aplZg==" - -GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "gemini/gemini-3.1-pro-preview", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", - expected_provider_headers={"content-type": "application/json", "x-goog-api-key": "synthetic-gemini-key"}, - expected_provider_request={ - "contents": [{"parts": [{"text": "Say hello."}], "role": "user"}], - "generationConfig": {"max_output_tokens": 1024}, - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], - "role": "model", - }, - "finishReason": "STOP", - "index": 0, - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 2, - "totalTokenCount": 115, - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10}], - "thoughtsTokenCount": 103, - "serviceTier": "standard", - }, - "modelVersion": "gemini-3.1-pro-preview", - "responseId": "5RjEaqfwC7XYqtsPp6TuoAY", - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "gemini/gemini-3.1-pro-preview", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 10, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 10, - "video_tokens": None, - }, - "output_tokens": 105, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 103, "text_tokens": 2}, - "total_tokens": 115, - "cost": None, - }, - "user": None, - "store": None, - }, -) diff --git a/tests/integration/translation/responses/bases/openai.py b/tests/integration/translation/responses/bases/openai.py index 2b120de6fe7..8e379e1019f 100644 --- a/tests/integration/translation/responses/bases/openai.py +++ b/tests/integration/translation/responses/bases/openai.py @@ -2,7 +2,8 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase - +from integration.translation.responses.templates import RESPONSES_NATIVE_RESPONSE, RESPONSES_REQUEST +from integration.translation.templates.openai import OPENAI_RESPONSES_REQUEST, OPENAI_RESPONSES_RESPONSE GPT_5_4_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -471,120 +472,148 @@ GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) - GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "openai/gpt-6-luna", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "openai/gpt-6-luna"}, expected_provider_endpoint="/responses", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, - expected_provider_request={ - "model": "gpt-6-luna", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST, "model": "gpt-6-luna"}, mock_provider_response={ - "id": "resp_05a79fb530391552006ac3f01c4cec87d0881e61192e1046ba", - "object": "response", - "created_at": 1791225884, - "status": "completed", - "access_programs": {"cyber": "daybreak_blue"}, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791225884, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6-luna", - "moderation": None, + "id": "resp_042cbd1ba73eb559006ad0239f6dc087d08fd7ff5fff84b06a", + "created_at": 1792025503, + "access_programs": {"cyber": "daybreak_blue"}, + "completed_at": 1792025506, "output": [ { - "id": "msg_05a79fb530391552006ac3f01cb60c87d082152e269d863aa4", + "id": "rs_042cbd1ba73eb559006ad023a0481487d0aea8635e4b717121", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COiaJAwBRbzsaBwqI_w56uTvcixfz0ImWwpaGwIbCDMXDOodeG_MedwCPLjagtmdVPeq1GAF21VbUncSo2oasMY26z31MMbmZQPKUXWfts9Dajf6g-11D0hZMUupKeeUOT_rcE_GBj6smH3d1PUf-FBMEsY1VQy0gr0am-GUa0dZjw3ukAX0K10gkPCrrqRgp7mlfEi9kAwXhlQ1WEVm7IdHdZnGBbWzKaIRDVkULZQlh9NHIFYCP-2OQbnmqK9vyUxLj3vYXIXGv9B0V3-9TPe8AfDg2tm3AH-tJ-XXJ8D6xDG4E-1avSe4vUrYggFM7_6ztYSrimWqtJ_PfvRQF4cZfcCDf3vxxzLreg3y0jtuZnEOejZYybCa72ZDBbI9OSgNvAZBkU6k9DLT_tQf9uCvN1vtoiGXu0Sk4yr1fw-hAnuEVJK_Tb_FfQBNEk7ILVK7UJaX1rZWTFe2GLhsp6ctz3Kp-c2w2JD5ZHvV22EDQ5DmP8ng9l_5COYQGSsS9ZLiyd06agQ3PvQvBTg5XnDL46DtZ1yfVkB1cuwwlUiWkKD6MQ-VifToT3s4aG0q1Bnl9b_AvtOM0lG-TblA-mdOaRDz3c9s4QfbQjt6Z75oQcNr5QkUujZoV4YiynIBpec5i2gr6Uv0SMJZZbzjY5PLiogj_gvUDxwVbWJkuMXjEPJ3WHWMjceJE4sHwuM95zWV80Ul-MSXknlBj-AZJGQ44_wl0765P--vLyq18jfKDCdorP3Lvq6lYxnXooYMnZ-lFgnWo92OHfX97d6FCyQ46ygzQaRjrhQwUyAioAZiFtugMnHUm7t-EfNb_C7TjLeyXLiJ9iVlGHIt17VJMJIIq_xrQeMCOHAxUKgUE21XkSwsVPdJ0I-4_OP5l3IwkiV0pctG1x-oYX7kCaIYDus-dJUehQNJWNEa_r6QPfn7txUoxMrNCCIyzRUfH3hgwCHudDJ26dj15RPGfiP3wQmX8oZKdy5VkQcNPWb4Gv6b-cjyI7U-8VmwlodUN5FlCj-LJbeL8C8Wxuf1LRzDW-kxNadEhxlQaDSq4nQfTnGARMcmlH8-XqZfXK-3hJD84xHuyjTHq7aBr85gM6vISoYOHf6d-5GIr3-z9cTXvE7tCuxoJ7DdZYWzRK0Bws6tCerRuuvlVzQhlS_CsPwoel-eQ4SJucxvFBI_RaA3LYi3GtLqUPSKTAW6QqnQvyK-GvGvfDFxlgHgByGB0sf3dBAYLFKwHYspVsA4JqX8LDXNLJzMl5_9OkEsOkOcnTdPQSwImve3XZeM8mZwcHWRXOq6viGM9kGcVBrGZoqg--bUJZshNp2EdQivXJ4APh0E17Ro0gUSvvPTrwwI4m3MS0YHACkf1MbidrTxeyI7fiuADGpLKuC2n83Xbs7xjln0p-IwylsHOKs5cd6Jlrjf1RjyECm9jLMexUpSx5iBxcoIRG5TvyJA3SicZU-_psosM-7ABr7sWWy3F8_e1Q9cIze10_6U7XwqZjIUls007qiE97TFUX7YTlP0zTUwy4xmaSHW94LjhjaJnle49iDdqyRQJCdJbGNHPAhFSHGT6Iyy5vPADzIqPYmASOWxJ_7-IX9LRZQxa2NCRdXou0yAvbSbuq9LL3xFhJQ9S0pvJkY02np0GpMRlWXQpzHlww1i8ptRTsyELKGlrVJdPsdKc2d9ZgHGPvGVxG-e7ESPnA9m1aLP7vVH8NP2qnPSOGQk4SDEw6rxlv5Kjkv56kAXzdKj2PvxuclfWTxG5Peo84gaMNwP_Mi6S4XSuNJt7RhNdTEdQPFIQ0bNcc3O5jnz_xDw76mF3YvRMm_kXxdaOsFP9GSfzaEY56CQ9VGOkziwqrPwWu6fDT7VpkVvi58UDDWJdnk3G-nvcxGHLERU1es-Qy6Pe2qWOdY-nU70YSqgVIj5MMyx6XMJcYMeJDi5831kzr3VgOAggT8K-yYibn4_K2M0OHO0oQW-sUKsljU08Eu60C4MLrSJB40qJGCgwuiFwfXZfKckRmGE-oE6X8uS1ZIgd3S7OSCpYXfNSW7PlRAobBB2ArFFtacVEeH4pjmgcbrjkXHCV_ddCHvo26aBXh_AiBgJkjQ-oqrOS3pgC9J9U-coE-yfNbTl_UcdfFfusG6tvMNV8rl9Pkao6-pG9PTM9E25XZj0rEHOW_Vj9JMqX_kTEP7AKTJr8VZH_fKPbYR-EKlf12hStW--S5P4ng1hDFRcXcYAxmb9panxXRD2mxBBkEUGd3TdG4lxR02K0f9DaDJH2_-u3l4WKQdcGh1ma7m5RYl94o794hvNm2moOazjhIUWseixhZRM0AtS50RHzxaBl3t-vp4QA657xwts-Ng2ERKH2gDR1IIpA9rCvYjRo4Xwif9mTB7hYggs6YqrUZiIHrfNth9Am5e3B5z11hnxgGHPC0Rq1DodR1oV-rmyXM_L8-UpwUrM3Xbg31E_k0p7vIjn635OjUZt9aSe-yXEexGaDpxP5rUHQW3A-X4lZ3a_cL_786Dyzr9Sgma1bM2rpi3j5XfhpQe-DZk4KMP0C_d9rEcSoC46DTNjksrQkb421C5xpRoyJQ2-iSZBplpXRWX3GKwuiGvdtxCdSRcqZ0JHEf3ScOvOVs4gLpLRi5NcTuofbQYAsOATXtDzSPGd07h42Mk65qttYHLwVkt--4WPAIdsgU4SX_P", + "summary": [], + }, + { + "id": "msg_042cbd1ba73eb559006ad023a198dc87d0a13500a8fdffc2d3", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to see how it handles zero, negative, " + "and non-integer quantities.", + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_042cbd1ba73eb559006ad023a20c5487d09f05bf43be3ac720", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":160}', + "call_id": "call_YeruxIVkbMog0O7xEjBn8luk", + "name": "read_file", + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 159, + "output_tokens_details": {"reasoning_tokens": 94}, + "total_tokens": 468, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, "id": ANY, - "created_at": 1791225884, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + "created_at": 1792025503, "model": "openai/gpt-6-luna", - "object": "response", "output": [ { - "id": "msg_05a79fb530391552006ac3f01cb60c87d082152e269d863aa4", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "id": "rs_042cbd1ba73eb559006ad023a0481487d0aea8635e4b717121", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COiaJAwBRbzsaBwqI_w56uTvcixfz0ImWwpaGwIbCDMXDOodeG_MedwCPLjagtmdVPeq1GAF21VbUncSo2oasMY26z31MMbmZQPKUXWfts9Dajf6g-11D0hZMUupKeeUOT_rcE_GBj6smH3d1PUf-FBMEsY1VQy0gr0am-GUa0dZjw3ukAX0K10gkPCrrqRgp7mlfEi9kAwXhlQ1WEVm7IdHdZnGBbWzKaIRDVkULZQlh9NHIFYCP-2OQbnmqK9vyUxLj3vYXIXGv9B0V3-9TPe8AfDg2tm3AH-tJ-XXJ8D6xDG4E-1avSe4vUrYggFM7_6ztYSrimWqtJ_PfvRQF4cZfcCDf3vxxzLreg3y0jtuZnEOejZYybCa72ZDBbI9OSgNvAZBkU6k9DLT_tQf9uCvN1vtoiGXu0Sk4yr1fw-hAnuEVJK_Tb_FfQBNEk7ILVK7UJaX1rZWTFe2GLhsp6ctz3Kp-c2w2JD5ZHvV22EDQ5DmP8ng9l_5COYQGSsS9ZLiyd06agQ3PvQvBTg5XnDL46DtZ1yfVkB1cuwwlUiWkKD6MQ-VifToT3s4aG0q1Bnl9b_AvtOM0lG-TblA-mdOaRDz3c9s4QfbQjt6Z75oQcNr5QkUujZoV4YiynIBpec5i2gr6Uv0SMJZZbzjY5PLiogj_gvUDxwVbWJkuMXjEPJ3WHWMjceJE4sHwuM95zWV80Ul-MSXknlBj-AZJGQ44_wl0765P--vLyq18jfKDCdorP3Lvq6lYxnXooYMnZ-lFgnWo92OHfX97d6FCyQ46ygzQaRjrhQwUyAioAZiFtugMnHUm7t-EfNb_C7TjLeyXLiJ9iVlGHIt17VJMJIIq_xrQeMCOHAxUKgUE21XkSwsVPdJ0I-4_OP5l3IwkiV0pctG1x-oYX7kCaIYDus-dJUehQNJWNEa_r6QPfn7txUoxMrNCCIyzRUfH3hgwCHudDJ26dj15RPGfiP3wQmX8oZKdy5VkQcNPWb4Gv6b-cjyI7U-8VmwlodUN5FlCj-LJbeL8C8Wxuf1LRzDW-kxNadEhxlQaDSq4nQfTnGARMcmlH8-XqZfXK-3hJD84xHuyjTHq7aBr85gM6vISoYOHf6d-5GIr3-z9cTXvE7tCuxoJ7DdZYWzRK0Bws6tCerRuuvlVzQhlS_CsPwoel-eQ4SJucxvFBI_RaA3LYi3GtLqUPSKTAW6QqnQvyK-GvGvfDFxlgHgByGB0sf3dBAYLFKwHYspVsA4JqX8LDXNLJzMl5_9OkEsOkOcnTdPQSwImve3XZeM8mZwcHWRXOq6viGM9kGcVBrGZoqg--bUJZshNp2EdQivXJ4APh0E17Ro0gUSvvPTrwwI4m3MS0YHACkf1MbidrTxeyI7fiuADGpLKuC2n83Xbs7xjln0p-IwylsHOKs5cd6Jlrjf1RjyECm9jLMexUpSx5iBxcoIRG5TvyJA3SicZU-_psosM-7ABr7sWWy3F8_e1Q9cIze10_6U7XwqZjIUls007qiE97TFUX7YTlP0zTUwy4xmaSHW94LjhjaJnle49iDdqyRQJCdJbGNHPAhFSHGT6Iyy5vPADzIqPYmASOWxJ_7-IX9LRZQxa2NCRdXou0yAvbSbuq9LL3xFhJQ9S0pvJkY02np0GpMRlWXQpzHlww1i8ptRTsyELKGlrVJdPsdKc2d9ZgHGPvGVxG-e7ESPnA9m1aLP7vVH8NP2qnPSOGQk4SDEw6rxlv5Kjkv56kAXzdKj2PvxuclfWTxG5Peo84gaMNwP_Mi6S4XSuNJt7RhNdTEdQPFIQ0bNcc3O5jnz_xDw76mF3YvRMm_kXxdaOsFP9GSfzaEY56CQ9VGOkziwqrPwWu6fDT7VpkVvi58UDDWJdnk3G-nvcxGHLERU1es-Qy6Pe2qWOdY-nU70YSqgVIj5MMyx6XMJcYMeJDi5831kzr3VgOAggT8K-yYibn4_K2M0OHO0oQW-sUKsljU08Eu60C4MLrSJB40qJGCgwuiFwfXZfKckRmGE-oE6X8uS1ZIgd3S7OSCpYXfNSW7PlRAobBB2ArFFtacVEeH4pjmgcbrjkXHCV_ddCHvo26aBXh_AiBgJkjQ-oqrOS3pgC9J9U-coE-yfNbTl_UcdfFfusG6tvMNV8rl9Pkao6-pG9PTM9E25XZj0rEHOW_Vj9JMqX_kTEP7AKTJr8VZH_fKPbYR-EKlf12hStW--S5P4ng1hDFRcXcYAxmb9panxXRD2mxBBkEUGd3TdG4lxR02K0f9DaDJH2_-u3l4WKQdcGh1ma7m5RYl94o794hvNm2moOazjhIUWseixhZRM0AtS50RHzxaBl3t-vp4QA657xwts-Ng2ERKH2gDR1IIpA9rCvYjRo4Xwif9mTB7hYggs6YqrUZiIHrfNth9Am5e3B5z11hnxgGHPC0Rq1DodR1oV-rmyXM_L8-UpwUrM3Xbg31E_k0p7vIjn635OjUZt9aSe-yXEexGaDpxP5rUHQW3A-X4lZ3a_cL_786Dyzr9Sgma1bM2rpi3j5XfhpQe-DZk4KMP0C_d9rEcSoC46DTNjksrQkb421C5xpRoyJQ2-iSZBplpXRWX3GKwuiGvdtxCdSRcqZ0JHEf3ScOvOVs4gLpLRi5NcTuofbQYAsOATXtDzSPGd07h42Mk65qttYHLwVkt--4WPAIdsgU4SX_P", + }, + { + "id": "msg_042cbd1ba73eb559006ad023a198dc87d0a13500a8fdffc2d3", + "content": [ + { + "annotations": [], + "text": "I’ll read `shop/validation.py` first to see how it handles " + "zero, negative, and non-integer quantities.", + "type": "output_text", + "logprobs": [], + } + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":160}', + "call_id": "call_YeruxIVkbMog0O7xEjBn8luk", + "name": "read_file", + "type": "function_call", + "id": "fc_042cbd1ba73eb559006ad023a20c5487d09f05bf43be3ac720", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "status": "completed", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -594,17 +623,16 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 159, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 94, "text_tokens": None}, + "total_tokens": 468, "cost": None, }, - "user": None, "store": True, "access_programs": {"cyber": "daybreak_blue"}, "background": False, "billing": {"payer": "developer"}, - "completed_at": 1791225884, + "completed_at": 1792025506, "frequency_penalty": 0.0, "max_tool_calls": None, "moderation": None, @@ -627,120 +655,104 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) - GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "openai/gpt-6.1-sol", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "openai/gpt-6.1-sol"}, expected_provider_endpoint="/responses", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, - expected_provider_request={ - "model": "gpt-6.1-sol", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST, "model": "gpt-6.1-sol"}, mock_provider_response={ - "id": "resp_05eee41d540f8798006ac3f01fff8887d0bf3cdc6e114644ae", - "object": "response", - "created_at": 1791225888, - "status": "completed", - "access_programs": None, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791225889, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6.1-sol", - "moderation": None, - "output": [ - { - "id": "msg_05eee41d540f8798006ac3f02102c887d08143f5010642dc50", - "type": "message", - "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", - "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, - }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", - "usage": { - "input_tokens": 19, - "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, - }, - "user": None, - "metadata": {}, }, expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, "id": ANY, - "created_at": 1791225888, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + "created_at": 1792025514, "model": "openai/gpt-6.1-sol", - "object": "response", "output": [ { - "id": "msg_05eee41d540f8798006ac3f02102c887d08143f5010642dc50", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "id": "rs_0207c20ae125feee006ad023abed4087d09d32a32ad2d3182c", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COvx3R_lOOaYJijiIokUKBq9tt_p3mhoPHnm4KR0Pj_t0wf1sP7WEDHfYSzylhHilYoKjU4guIz-TAqBV6HNUpIx5ggp3aukKEW6zks2ERT3a40sCFFdUdKHk6py5trFuwYn6lHLwifwDkd7rI7MKIP7C7pZSHnXh8cmfk0t96p4nBplQ93C4m917Kyx6PqvWSxPP78h5s_RMEdhkWRU20xMhminpIBNArOkPYT4xsmBEuONlndM5-DHG7ZeLk2yMZw8b_hxk2LZ4mC66b5RHfJKVKfw4egn4K8tkr8mQSR4bnjpY56DsIRj4fjSH5GpoDZyolFFBh1BNkGMoVsicye_4gZn0eELuOWxCw7lVVsMpIzTLw8CHJkCo_8vKtNCFSZcKjOPKDDfX1bsFJ7LMonmqijHPV_kZi_YXjVNNJBhJLiGjYrVUFZzjS0OKQx-RnZf5luO3UM2Ff3Ilvx1eQtuWlsvJjPBRwObzk5zlPEdYwoh5OjUREbgGb4H71Q7A9YYjUIms2h5e13Z5s8X0onRVr9SUcSbbMA0FXWn6QGQ5qgiQV56xec7xk62SrxKBXeILgBttBf68YfLAbdINCPL89G5Z1X2toQLlFM7e0jKYUOvSmpWyPLZ3q1040oDf2Q92v6NjOwk4WlLtZF_4xrZ7bEIWmea7zcDEnsvNBRdKaUbeUOm0SfRuIH4gQfbbueSqM3i12eK4TQhTbJ9VivUUovUv7pCKvizdQuunf2MMXWhq79wLYQYXIleQ06E9hbx6QT-Ue5ELQVC3NSLhXiwucOSqd-NZFDQA5wgW2Uaq_-bFYzNOPwtArfaqnHOILmr5mDvWCrlnLs-DUPHU-DtzYH7Eu1VSlcEZLaZX4vC9ui3x2aOpTtmVCILZpw7MkouVxSR4rFE48doVoCamJHhey-TZ7OMl0A5JdacTt6oUE8Z2HEcuVh9yGAZe_Q-YB3p4egLbxO8ICWUkwSSNgrPq3WRbT9sbT-Ko7N7KDAfgyvzzdEYMKDmci6GBrVsAiscc97-gxbhVhlrMyIXmgTluW07o591YjlOkvERXeJobEfby-VUJCu0cHd0nMudzrhCpHIhU3aUMkk9GMlSNDPFnKeV-gylycLhYRb0xo1kXMRom82fKmvUgO5Wh_BhsCdeGaVQ2bm5wyLT4x9VhEe-uOwaYG02XxoSiIFqqSrf0J8auFHIRpUTKzNWMK_GcIaU6o3ON0wOLKaga3wcyrGkG6wtvGkNLCrvmDIXg0bmvFGZrmybQvys_u636NXVS-Y9irqUv2ghu5tyKI_F8uwibi9JkKa5XcotQGbAToTE8qaHfowZOdTyjIKrU3BjL_Hs5M41KXwrNkk_X5ttJxEPW1qCjyusFb9yeGa9wzHMsp_VO0ysdO8L2KSRAY4LHpGI175VxUmnSg66PtCPln0OYxWsX2FWE1A7B_la662tO4VoS3bcslVZreRWFDJ_7X8qKx2fStxd77B7JkG1ZElxlZYlBRN5_Lo_w1yM9SvqHzzewQYEprSA1gltjxAbIzy7xXReaT86PNI8ojSLfXOkXa6Cybu3hIxsIomkZwTtAEx85BoIDn2B8oTp744Lwkwn4ualAfV9TebiqOWCUYp47fX9rimte71X6Hnvf7tWEBZMMwSb0pTqY0boMOS0w3XKKyEVRNyCoGWMgyvhAxk9E5iao5N6i5W3aSel27QwsUaE4YDGKKMKSryH3BqP6X7d0kdUcAx-nd5yudVbePQyc-UC7EbISj5QXY8g6cTOy-IkwpBe68DvfcNGIhuaH6UscqrewXQjKZFJC_BJSl-ICOfMw-FCTumzCDXPqTeLBPHf1g5kd5lBd6HLPUldHtJfD48V12KCRtaoksZ4g2NibuClnmZRpnDMPDUr7r_D8-RfoLEiJeh8SbOSnLvMPVAJIB95F73B-UX5u0e0aWeyfG5QjBq-kB7vVGsRiMyHEVQW810xo2qULVIyVO03s7rAWwTxFY-EHmKkaAIbKU2b-qAasytni7f1Aq61OSIYqSCRdFhpUX6HpnQeCFC_s6_Q9P3CFx120ZaTB_HPeRJO6dhVSJ3nL3PiNi8sCnhRiNIXkJfm9l1TxRS8ok9s1NY8oi4Y98J4Ay52EkX6f9F9FtWHNu4GX_Z50jDNnVTVqQo3L3slpwxECsTAEU-0MsmIa-1RqJ00VqBLKtIDVbbnQ==", + }, + { + "id": "msg_0207c20ae125feee006ad023ade23487d093e272f84cb35723", + "content": [ + { + "annotations": [], + "text": "I’ll read `shop/validation.py` first to check how it handles " + "zero, negative, and non-integer quantities.", + "type": "output_text", + "logprobs": [], + } + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_S4WtfrQwa6ZLEUZf1BVtA8ON", + "name": "read_file", + "type": "function_call", + "id": "fc_0207c20ae125feee006ad023ae893c87d08068561e36e665ad", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "status": "completed", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -750,17 +762,16 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 118, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 53, "text_tokens": None}, + "total_tokens": 427, "cost": None, }, - "user": None, "store": True, "access_programs": None, "background": False, "billing": {"payer": "developer"}, - "completed_at": 1791225889, + "completed_at": 1792025519, "frequency_penalty": 0.0, "max_tool_calls": None, "moderation": None, @@ -786,116 +797,144 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "openai/gpt-6-astra", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "openai/gpt-6-astra"}, expected_provider_endpoint="/responses", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, - expected_provider_request={ - "model": "gpt-6-astra", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST, "model": "gpt-6-astra"}, mock_provider_response={ - "id": "resp_09642d0a94fde0c6006ad016cb1e4c87d0bf52dd7767aa00c2", - "object": "response", - "created_at": 1792022219, - "status": "completed", - "access_programs": None, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1792022220, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6-astra", - "moderation": None, + "id": "resp_0552d3d37f56f8e8006ad02393df1487d08cd567c9669c6b15", + "created_at": 1792025491, + "completed_at": 1792025495, "output": [ { - "id": "msg_09642d0a94fde0c6006ad016cbe8bc87d0a9da8d51688a029e", + "id": "rs_0552d3d37f56f8e8006ad02395050087d0954cb16260eceabd", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COXbYvBd4No2o1mKicIiH9j9eRzT8OGbkVyS-7zuUXSU6YKYudPsLUl2WsonyL61adFZSVoTsnF028sqP7sbKTb4PJS8w6eMYLETqeIxL20JUkHB93j798SzS50qm_X4_cSDnH7mh5rcnTNOcdk9BAlrjFTKuK_ARzCswFDmqH2mk4zZiD767r4iXv0QyxKxsEWq-exnoaJOw39XRFQQA_1ATo1J6dP6AmQkpmhQKlIY8NPKwNiR34X2jpN9yskz9PeJs_a1jA4R1m2Xh7ISAS2gmKX3vaBJL4sM8K7sV7KgAuSboorGK9N-L0737_zPJSKvnqI0-2uovkBaYzKwpfpUGs5zFK6zE6_eQJ5JLCCFUfarz_iIyQXMExN5jKoqD7rfP8eMjEAuW2OqWVrL6i-cnBp96G8erGV7y2uD0_iIbEJ6dj2dWE9u81jrpMilRDo4VRDeff7kpWCNYdvn_LRzgKwhDSLHunrZKp-yi3056kNaNDiw8ra9_BwrfEAduOnMvbdLNSCRKJ8WbkL4JNZU3KOfsDCeoqXRWHsJOgUXMjruTgDBKJccKjYF5kzhNRxeb4dAKZqKgEqvCV3QX9wQhXVEnqVct3KHU0QDQLYTs4EwfBTf_ReVWYtIowJrRZrtjW6Qdq9z7v6HY3-wiRkwnDoXJriySRuZviX3y3m4cwCe63gc3DqFvS22UWXk4cIw3XCdsJ5l-uJNWs_-FXUvGv4H2l2iunVPMWBVlorL-SdkGa9ftdpb7gPtZD8gxjryYO7vJO7Sjk1K7A0gqAZHvIwJMdFJxED0yxpPv7hEqw_3DFQDJr3elnjmOwHVBueWYggjN-6oyYJwQOo0o2_M3pyUsOuS8eGIm2TEL8LKXl-QeHbUbISQm_ZZYcUqwnhvTRlslx2Z9dTwgN5u2HwL1CArcAQ_j1pyAJIKoHCgIp7EWgS0zsgaiU1wJRs3aw1wzUQJ5EGKKkle3EIDJnhLA9l_UOQ1xAchhIExf76-eGuAiEQi7v0Gz6SMiLSlUH2xh7fyve3_mLuOYGidn73xCr91y8TjuAxRYiC0oJP6jYgUtVZ0MpWHYFwhdgtuVM-t4c3XNrNLpBqyWZl7ubi4Jy513NTqr4m4ExktN-5HkKkjb5P-35WccTeXVu8o_NN42ysHO3_mPwPe6-Jr3k-7CrwCPiEhR4rStyuitp_PeGBDPCVgW6vj7urfDe11zAZ84shO4yBVtG26c_CqtLq7MWxKQVrTLx1911InHrIDf7BJjFceVzfYiGEplBNT3OzBDJsbjmZYB_ppdiZeyaDSML93ysldny_6KYKx7cGcdHPyU4hVii-nlr0FdH_v9hngcWj4RUO6k9N9W3XHcmxoNLbX6calO2UMD4O94IiS-pLd-zpMdm-Bo5ncucxfMMc3WuCpVLDIpgzrfF_5F_9BYfO1PdYhCGclFfVij6znXR9CuKOXHhcUAG0kmUb_u_lyTpixNv-B4AJLgI1eONjazeSbGHxRwmVOFF11fIY28T3E0kHO940CxteM3eQKCxEUJmpOc6o1iPuy5jr8NOUrW_Bvp-QF1yS4SZHrSqvPUkUjpjzTy41KrbBgHqtoJ_8GwUrTLvUSPsec1nrErNMJE1oZB_6PDP477aer4gbZOil6N73kGVl2-2HeWMgOFh3qiosi71SsJJ9XmGV3bU7zPjLhtKsyoSOTlhmep-7Rs81V8MSvzhBqWBRA-EI2a07Cc24wsIRDgf366sg9tbbsOK3OdgXG3a3vyQNbflzbMnkRyqzzDCDmkWzRz_BBXydi19tKPPFuLpwEZ12oZE7bfy9uPaAfHMPNFPuZhcV3vVNaCgOX-hNht5RqRQTO8esmOemEtN0hpOEnc7cgS8xGNjMQBzvDnvXEU1eJSnwSkdWwUvlDz_OTwgorvzLcr20cHsZTCL9nP4F_-12eWJvn37at6uSJN3aNrDyWoUe9sSee0NiI3ne2seWkqi6wF8_mOxheeQBQcZtLL0agyGXg_YrcnvqjcUrOyjLNBEutMLMVCjVehyhKmYPh-WHzajm9d8hMD91juCft6ufCe52APO1TmNxqUuphA-gFT4Xo1crf7lx6QJVp8em2NmfNMFDVvx-xzi84epgkulTk95GLqH1ZtsXNt-WBinvx737nLtsA06iA9c946KfNTZxARfe9cb1VVjNo0cnFpTkO6GBVwZC83g540pmAdi57yjjFmTk7uf8sVF-ZQ2LheqVDs2VTl1Z74NJIFPH1tapSawFDL6jhFgekEWOAsUlTshMrhsBVfqY1rBw5JFQKOXfz7H33_KlmBBHl0EkRrqy75mHxuVRW9nxisCXg85lvEOAiOIizLu9HMEJvpmzDdkcZSrc3PvaYVRup5y6w2Pwr4Dj37t1y_ls3YPnnoVkyiXdCoP_wHxgFeUbfNAfKisQxbRy2d-VG_5hRul3F0AqGhr86rT0X1eTb752Up2eFoL6CfAtc5h-oIck8JdJbE34ELHCXg36hx1o1I-WnnHemhIJZGDDfyvGx-KQdoz55SkP4FMxsc-4SnQeOa7nn0-KruRaNTiq6NDhjCILlEZ6M6kIN7nDWYgZgsuWJCKqEoHbi4yodzpUpH9jr0kVlaP4R74WibiSmS-fdQIKnbh9tUcrF_rzbc1VIxI34ODmStok02kmDeao33n8CSb4WA_RW-glw3D138j-HGprBOHR1l0DdcOX0PPk1cvMzsz87b-RgnVVGmv-DqWKL1GgoUItHUp22wpOpPWLb6W71xPHmc2e0CuiqSsAd2Qz1lOYXImJk90BRX-09V4FOB7G9MIGZ5EJ-J4lirYTM5DZrNuU9RFGvi0Gv1EJa_9ejTm6kArC1D_SAXJqoLL8q647FxfyGzJPyhgQn3BbXfclnJRmMFeoZA==", + "summary": [], + }, + { + "id": "msg_0552d3d37f56f8e8006ad02396d75087d0a2c8b7137ba77a35", "type": "message", "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], - "phase": "final_answer", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles zero, " + "negative, and non-integer quantities.", + } + ], + "phase": "commentary", "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", + { + "id": "fc_0552d3d37f56f8e8006ad02397444087d09ede3b901e93cb70", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_wVrKXoGNLx7ClNV8cmG0E8pP", + "name": "read_file", + }, + ], "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, + "output_tokens": 177, + "output_tokens_details": {"reasoning_tokens": 112}, + "total_tokens": 486, }, - "user": None, - "metadata": {}, }, expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, "id": ANY, - "created_at": 1792022219, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + "created_at": 1792025491, "model": "openai/gpt-6-astra", - "object": "response", "output": [ { - "id": "msg_09642d0a94fde0c6006ad016cbe8bc87d0a9da8d51688a029e", - "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "id": "rs_0552d3d37f56f8e8006ad02395050087d0954cb16260eceabd", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COXbYvBd4No2o1mKicIiH9j9eRzT8OGbkVyS-7zuUXSU6YKYudPsLUl2WsonyL61adFZSVoTsnF028sqP7sbKTb4PJS8w6eMYLETqeIxL20JUkHB93j798SzS50qm_X4_cSDnH7mh5rcnTNOcdk9BAlrjFTKuK_ARzCswFDmqH2mk4zZiD767r4iXv0QyxKxsEWq-exnoaJOw39XRFQQA_1ATo1J6dP6AmQkpmhQKlIY8NPKwNiR34X2jpN9yskz9PeJs_a1jA4R1m2Xh7ISAS2gmKX3vaBJL4sM8K7sV7KgAuSboorGK9N-L0737_zPJSKvnqI0-2uovkBaYzKwpfpUGs5zFK6zE6_eQJ5JLCCFUfarz_iIyQXMExN5jKoqD7rfP8eMjEAuW2OqWVrL6i-cnBp96G8erGV7y2uD0_iIbEJ6dj2dWE9u81jrpMilRDo4VRDeff7kpWCNYdvn_LRzgKwhDSLHunrZKp-yi3056kNaNDiw8ra9_BwrfEAduOnMvbdLNSCRKJ8WbkL4JNZU3KOfsDCeoqXRWHsJOgUXMjruTgDBKJccKjYF5kzhNRxeb4dAKZqKgEqvCV3QX9wQhXVEnqVct3KHU0QDQLYTs4EwfBTf_ReVWYtIowJrRZrtjW6Qdq9z7v6HY3-wiRkwnDoXJriySRuZviX3y3m4cwCe63gc3DqFvS22UWXk4cIw3XCdsJ5l-uJNWs_-FXUvGv4H2l2iunVPMWBVlorL-SdkGa9ftdpb7gPtZD8gxjryYO7vJO7Sjk1K7A0gqAZHvIwJMdFJxED0yxpPv7hEqw_3DFQDJr3elnjmOwHVBueWYggjN-6oyYJwQOo0o2_M3pyUsOuS8eGIm2TEL8LKXl-QeHbUbISQm_ZZYcUqwnhvTRlslx2Z9dTwgN5u2HwL1CArcAQ_j1pyAJIKoHCgIp7EWgS0zsgaiU1wJRs3aw1wzUQJ5EGKKkle3EIDJnhLA9l_UOQ1xAchhIExf76-eGuAiEQi7v0Gz6SMiLSlUH2xh7fyve3_mLuOYGidn73xCr91y8TjuAxRYiC0oJP6jYgUtVZ0MpWHYFwhdgtuVM-t4c3XNrNLpBqyWZl7ubi4Jy513NTqr4m4ExktN-5HkKkjb5P-35WccTeXVu8o_NN42ysHO3_mPwPe6-Jr3k-7CrwCPiEhR4rStyuitp_PeGBDPCVgW6vj7urfDe11zAZ84shO4yBVtG26c_CqtLq7MWxKQVrTLx1911InHrIDf7BJjFceVzfYiGEplBNT3OzBDJsbjmZYB_ppdiZeyaDSML93ysldny_6KYKx7cGcdHPyU4hVii-nlr0FdH_v9hngcWj4RUO6k9N9W3XHcmxoNLbX6calO2UMD4O94IiS-pLd-zpMdm-Bo5ncucxfMMc3WuCpVLDIpgzrfF_5F_9BYfO1PdYhCGclFfVij6znXR9CuKOXHhcUAG0kmUb_u_lyTpixNv-B4AJLgI1eONjazeSbGHxRwmVOFF11fIY28T3E0kHO940CxteM3eQKCxEUJmpOc6o1iPuy5jr8NOUrW_Bvp-QF1yS4SZHrSqvPUkUjpjzTy41KrbBgHqtoJ_8GwUrTLvUSPsec1nrErNMJE1oZB_6PDP477aer4gbZOil6N73kGVl2-2HeWMgOFh3qiosi71SsJJ9XmGV3bU7zPjLhtKsyoSOTlhmep-7Rs81V8MSvzhBqWBRA-EI2a07Cc24wsIRDgf366sg9tbbsOK3OdgXG3a3vyQNbflzbMnkRyqzzDCDmkWzRz_BBXydi19tKPPFuLpwEZ12oZE7bfy9uPaAfHMPNFPuZhcV3vVNaCgOX-hNht5RqRQTO8esmOemEtN0hpOEnc7cgS8xGNjMQBzvDnvXEU1eJSnwSkdWwUvlDz_OTwgorvzLcr20cHsZTCL9nP4F_-12eWJvn37at6uSJN3aNrDyWoUe9sSee0NiI3ne2seWkqi6wF8_mOxheeQBQcZtLL0agyGXg_YrcnvqjcUrOyjLNBEutMLMVCjVehyhKmYPh-WHzajm9d8hMD91juCft6ufCe52APO1TmNxqUuphA-gFT4Xo1crf7lx6QJVp8em2NmfNMFDVvx-xzi84epgkulTk95GLqH1ZtsXNt-WBinvx737nLtsA06iA9c946KfNTZxARfe9cb1VVjNo0cnFpTkO6GBVwZC83g540pmAdi57yjjFmTk7uf8sVF-ZQ2LheqVDs2VTl1Z74NJIFPH1tapSawFDL6jhFgekEWOAsUlTshMrhsBVfqY1rBw5JFQKOXfz7H33_KlmBBHl0EkRrqy75mHxuVRW9nxisCXg85lvEOAiOIizLu9HMEJvpmzDdkcZSrc3PvaYVRup5y6w2Pwr4Dj37t1y_ls3YPnnoVkyiXdCoP_wHxgFeUbfNAfKisQxbRy2d-VG_5hRul3F0AqGhr86rT0X1eTb752Up2eFoL6CfAtc5h-oIck8JdJbE34ELHCXg36hx1o1I-WnnHemhIJZGDDfyvGx-KQdoz55SkP4FMxsc-4SnQeOa7nn0-KruRaNTiq6NDhjCILlEZ6M6kIN7nDWYgZgsuWJCKqEoHbi4yodzpUpH9jr0kVlaP4R74WibiSmS-fdQIKnbh9tUcrF_rzbc1VIxI34ODmStok02kmDeao33n8CSb4WA_RW-glw3D138j-HGprBOHR1l0DdcOX0PPk1cvMzsz87b-RgnVVGmv-DqWKL1GgoUItHUp22wpOpPWLb6W71xPHmc2e0CuiqSsAd2Qz1lOYXImJk90BRX-09V4FOB7G9MIGZ5EJ-J4lirYTM5DZrNuU9RFGvi0Gv1EJa_9ejTm6kArC1D_SAXJqoLL8q647FxfyGzJPyhgQn3BbXfclnJRmMFeoZA==", + }, + { + "id": "msg_0552d3d37f56f8e8006ad02396d75087d0a2c8b7137ba77a35", + "content": [ + { + "annotations": [], + "text": "I’ll read `shop/validation.py` first to check how it handles " + "zero, negative, and non-integer quantities.", + "type": "output_text", + "logprobs": [], + } + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_wVrKXoGNLx7ClNV8cmG0E8pP", + "name": "read_file", + "type": "function_call", + "id": "fc_0552d3d37f56f8e8006ad02397444087d09ede3b901e93cb70", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "status": "completed", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -905,17 +944,199 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 177, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 112, "text_tokens": None}, + "total_tokens": 486, "cost": None, }, - "user": None, "store": True, "access_programs": None, "background": False, "billing": {"payer": "developer"}, - "completed_at": 1792022220, + "completed_at": 1792025495, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + +GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "openai/gpt-5.6-terra"}, + expected_provider_endpoint="/responses", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST, "model": "gpt-5.6-terra"}, + mock_provider_response={ + **OPENAI_RESPONSES_RESPONSE, + "model": "gpt-5.6-terra", + "id": "resp_01818bf6930dd01f006ad0238a1f4087d0a5a47aea04e9cd47", + "created_at": 1792025482, + "access_programs": {"cyber": "daybreak_blue"}, + "completed_at": 1792025483, + "output": [ + { + "id": "rs_01818bf6930dd01f006ad0238abb8c87d0be1dc53bb1de580f", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COL8ir7ZwXbPOmVEOgRTL-aqfraf_1ghm0mkA8sdY9ksq7Sk_hwTMNcBH1zIfS7hFMRLuTRtwXIntfzRjxwisfCfN1fr76Fy_Nx9M0lEUQjPrTL0hgfAjCYoiOptHRO-v5EKTApgnrKVxPxuwDMYQ3GdGAFmN15SNSaWaqpYRbP96XmcvnnMZHLE-BtfbVxtdxSXpVuGLgfCC-11NKJSIG7kZHKi0iSsQBs4tG_Ga3iy1ZDHvrmgduDHR357yfT-Zdufjgm_cQ_KgmSXzcIvzS4Fv3wwTiUzH1SpFfW58Bbbr7gf6fYeLyszpMAMAQePBntjXdNSFV3-DmtKz8HPcZ2UjeAkwQkBWLsJfJoS7XTKP4U2xriGSvVpoCV4vx-GSAGEZGyOt3TyY8jAqOPMViSPGOLWiId0upWOolJUV_nDqRGjKlk8K-k4b_B2k0RiGD6AbUi3_8Ru2i1V0rkA451uOideERDk4LPCVJlgIwtGXOy-jySI17HeYNBjrNtH1mFojojsjt6KIaj3a38r6Xh0fvUfQZAKJJXVqmSIgMebFOI4iiesJxDuVK2qlBTS3mE_FNAfw3AkmKqRUyvEARYNJU_0Yt31EEFAED3xJ-KpSzx-WAfJec-kIT-PiRxvod_PKXObT2u5kJwy9Z8FLj37K1mWYKHHpY9dYwnm0z9o49S4K_f-fIgoGzDtRybAPrglZlDFmf_RSSrVCGo-h_F4Z07wme4zUpnwbGq5I0KmNZVa3d_JlTXpjEPsyDHhK6VHGOvKvOpESgH-7zekvV2_NLheXsAKqpCe3eZ_yI5FkYma5Uiw5lE30GuCT7Dm__eoHllH1AajuIJMvUQ4kF9d1RPXCSk9l9WE34IesApxU--bT1wd1rqedZSsgOW9AkKAq1NC9QEhXG-83_7R0wuUcDeCSdWs4vWqJ0YDjIWEg9LliHm5fypEOW-qftdlBoDWzdqwqMu0XEJvTT1IXtZTNFhekrTgTLpaFJtGw6T8bsaY4csZnb-0jQ9IZyHaRRwPX8JKb3ajvdq_33v_8kLGrVRYjZmd2MwFf2ysaNu_Ob7mNd2UdopntQ98XCbPAkzvvsTPfRYVs4Nf8fg5vAd6QvPy7xoRYxrvQGkppeOVbcPI-EuJSqN_WhTmbXKOpl69MDDyR7xLfuIYLuAbiSO1lS74JuhIgyGkVvO-hUUxL2utZhZdjr_N8AAg0CTKxuMlURkfXRVWrSpm7pm_u-0Xmi4XkrMDb1he5vZmgebI7KiQGkX_jKISBrK0fLwNYWJwY_6cF0QjiIvUt2gucGr7GBUEKeIo3r7oPPEZ3IzBWbiVWHPpjDDJdtliwDAxtzhd-FQwq6N3Rnmj1_4HiUFT3Akj61CYaKXj4IUyamTUdUc3cLcXge2Drv3Kno0ysPf-EHTm0syMRk8URfO9WnFPnjJGH0hIsffSkApE0WJ0Hk8wdzZGvP0QnRZ5lLlEtOBftxW8_-9D457g9kUDiBbnYqOekuQKotQfWfrOKNcEuQeyV1wV1HNrbS4T9Ul6gKFgkDk-wWaciavm2eElMM13OjXJpPsnZpat3pCXxKqrmFLiB4G4NFASz0RzowbnaPoc6OEv2j-c8QzpXX0ftVM0jVsTnl_izTcEobOjU7ipFLBBalJN3aq9K8OYNPKE0fegSQ5ywtERUqk4-X83ggwpDBBZe8tfkPf0KxwRyVmExW4kcepm8zZUzLxeZbipirXd3awzxaD5q-NiN1tvZC7bqb3-cy5du0H_7OScR2QGy7Wt3u2m3zixab8j1foXw6vvBQgV31sTL5RSdZlFtbyI1yu2T9SeOLzfknx14nUCWGyZsLpP3MSCmAOTlHFF6HMa-hnkoWrCtJBuWdq2BuQ9Q==", + "summary": [], + }, + { + "id": "msg_01818bf6930dd01f006ad0238b00a487d0aaa092bcd2431bed", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I need `shop/validation.py` next to inspect the validation branches for " + "zero, negative, and non-integer inputs.", + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_01818bf6930dd01f006ad0238b33dc87d0866bf763c4861a74", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_FI1PXRT2XNjueqLOd8GliYTF", + "name": "read_file", + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 100, + "output_tokens_details": {"reasoning_tokens": 35}, + "total_tokens": 409, + }, + }, + expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, + "id": ANY, + "created_at": 1792025482, + "model": "openai/gpt-5.6-terra", + "output": [ + { + "id": "rs_01818bf6930dd01f006ad0238abb8c87d0be1dc53bb1de580f", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COL8ir7ZwXbPOmVEOgRTL-aqfraf_1ghm0mkA8sdY9ksq7Sk_hwTMNcBH1zIfS7hFMRLuTRtwXIntfzRjxwisfCfN1fr76Fy_Nx9M0lEUQjPrTL0hgfAjCYoiOptHRO-v5EKTApgnrKVxPxuwDMYQ3GdGAFmN15SNSaWaqpYRbP96XmcvnnMZHLE-BtfbVxtdxSXpVuGLgfCC-11NKJSIG7kZHKi0iSsQBs4tG_Ga3iy1ZDHvrmgduDHR357yfT-Zdufjgm_cQ_KgmSXzcIvzS4Fv3wwTiUzH1SpFfW58Bbbr7gf6fYeLyszpMAMAQePBntjXdNSFV3-DmtKz8HPcZ2UjeAkwQkBWLsJfJoS7XTKP4U2xriGSvVpoCV4vx-GSAGEZGyOt3TyY8jAqOPMViSPGOLWiId0upWOolJUV_nDqRGjKlk8K-k4b_B2k0RiGD6AbUi3_8Ru2i1V0rkA451uOideERDk4LPCVJlgIwtGXOy-jySI17HeYNBjrNtH1mFojojsjt6KIaj3a38r6Xh0fvUfQZAKJJXVqmSIgMebFOI4iiesJxDuVK2qlBTS3mE_FNAfw3AkmKqRUyvEARYNJU_0Yt31EEFAED3xJ-KpSzx-WAfJec-kIT-PiRxvod_PKXObT2u5kJwy9Z8FLj37K1mWYKHHpY9dYwnm0z9o49S4K_f-fIgoGzDtRybAPrglZlDFmf_RSSrVCGo-h_F4Z07wme4zUpnwbGq5I0KmNZVa3d_JlTXpjEPsyDHhK6VHGOvKvOpESgH-7zekvV2_NLheXsAKqpCe3eZ_yI5FkYma5Uiw5lE30GuCT7Dm__eoHllH1AajuIJMvUQ4kF9d1RPXCSk9l9WE34IesApxU--bT1wd1rqedZSsgOW9AkKAq1NC9QEhXG-83_7R0wuUcDeCSdWs4vWqJ0YDjIWEg9LliHm5fypEOW-qftdlBoDWzdqwqMu0XEJvTT1IXtZTNFhekrTgTLpaFJtGw6T8bsaY4csZnb-0jQ9IZyHaRRwPX8JKb3ajvdq_33v_8kLGrVRYjZmd2MwFf2ysaNu_Ob7mNd2UdopntQ98XCbPAkzvvsTPfRYVs4Nf8fg5vAd6QvPy7xoRYxrvQGkppeOVbcPI-EuJSqN_WhTmbXKOpl69MDDyR7xLfuIYLuAbiSO1lS74JuhIgyGkVvO-hUUxL2utZhZdjr_N8AAg0CTKxuMlURkfXRVWrSpm7pm_u-0Xmi4XkrMDb1he5vZmgebI7KiQGkX_jKISBrK0fLwNYWJwY_6cF0QjiIvUt2gucGr7GBUEKeIo3r7oPPEZ3IzBWbiVWHPpjDDJdtliwDAxtzhd-FQwq6N3Rnmj1_4HiUFT3Akj61CYaKXj4IUyamTUdUc3cLcXge2Drv3Kno0ysPf-EHTm0syMRk8URfO9WnFPnjJGH0hIsffSkApE0WJ0Hk8wdzZGvP0QnRZ5lLlEtOBftxW8_-9D457g9kUDiBbnYqOekuQKotQfWfrOKNcEuQeyV1wV1HNrbS4T9Ul6gKFgkDk-wWaciavm2eElMM13OjXJpPsnZpat3pCXxKqrmFLiB4G4NFASz0RzowbnaPoc6OEv2j-c8QzpXX0ftVM0jVsTnl_izTcEobOjU7ipFLBBalJN3aq9K8OYNPKE0fegSQ5ywtERUqk4-X83ggwpDBBZe8tfkPf0KxwRyVmExW4kcepm8zZUzLxeZbipirXd3awzxaD5q-NiN1tvZC7bqb3-cy5du0H_7OScR2QGy7Wt3u2m3zixab8j1foXw6vvBQgV31sTL5RSdZlFtbyI1yu2T9SeOLzfknx14nUCWGyZsLpP3MSCmAOTlHFF6HMa-hnkoWrCtJBuWdq2BuQ9Q==", + }, + { + "id": "msg_01818bf6930dd01f006ad0238b00a487d0aaa092bcd2431bed", + "content": [ + { + "annotations": [], + "text": "I need `shop/validation.py` next to inspect the validation " + "branches for zero, negative, and non-integer inputs.", + "type": "output_text", + "logprobs": [], + } + ], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_FI1PXRT2XNjueqLOd8GliYTF", + "name": "read_file", + "type": "function_call", + "id": "fc_01818bf6930dd01f006ad0238b33dc87d0866bf763c4861a74", + "namespace": None, + "status": "completed", + }, + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], + "top_p": 0.98, + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 309, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 100, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 35, "text_tokens": None}, + "total_tokens": 409, + "cost": None, + }, + "store": True, + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1792025483, "frequency_penalty": 0.0, "max_tool_calls": None, "moderation": None, diff --git a/tests/integration/translation/responses/bases/openai_responses.py b/tests/integration/translation/responses/bases/openai_responses.py index 9505d75dcd9..ac92713f65f 100644 --- a/tests/integration/translation/responses/bases/openai_responses.py +++ b/tests/integration/translation/responses/bases/openai_responses.py @@ -2,6 +2,8 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import RESPONSES_NATIVE_RESPONSE, RESPONSES_REQUEST +from integration.translation.templates.openai import OPENAI_RESPONSES_REQUEST, OPENAI_RESPONSES_RESPONSE GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -161,116 +163,101 @@ GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "openai/responses/gpt-6.1-sol", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "openai/responses/gpt-6.1-sol"}, expected_provider_endpoint="/responses", - expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"}, - expected_provider_request={ - "model": "gpt-6.1-sol", - "input": "Say hello.", - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", }, + expected_provider_request={**OPENAI_RESPONSES_REQUEST, "model": "gpt-6.1-sol"}, mock_provider_response={ - "id": "resp_069a914d6d8c0f52006ac3f04d744887d0a3f54d9df9741033", - "object": "response", - "created_at": 1791225933, - "status": "completed", - "access_programs": None, - "background": False, - "billing": {"payer": "developer"}, - "completed_at": 1791225934, - "error": None, - "frequency_penalty": 0.0, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "max_output_tokens": 64, - "max_tool_calls": None, + **OPENAI_RESPONSES_RESPONSE, "model": "gpt-6.1-sol", - "moderation": None, - "output": [ - { - "id": "msg_069a914d6d8c0f52006ac3f04e984087d095c875b7ae7b5542", - "type": "message", - "status": "completed", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}], - "phase": "final_answer", - "role": "assistant", - } - ], - "parallel_tool_calls": True, - "presence_penalty": 0.0, - "previous_response_id": None, - "prompt_cache_key": None, - "prompt_cache_retention": "24h", - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "safety_identifier": None, - "service_tier": "default", - "store": True, - "temperature": 1.0, - "text": {"format": {"type": "text"}, "verbosity": "medium"}, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, - "total_tokens": 0, - }, - "web_search": {"num_requests": 0}, - }, - "tools": [], - "top_logprobs": 0, - "top_p": 0.98, - "truncation": "disabled", - "usage": { - "input_tokens": 19, - "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 25, - }, - "user": None, - "metadata": {}, }, expected_litellm_response={ + **RESPONSES_NATIVE_RESPONSE, "id": ANY, - "created_at": 1791225933, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + "created_at": 1792025514, "model": "openai/responses/gpt-6.1-sol", - "object": "response", "output": [ { - "id": "msg_069a914d6d8c0f52006ac3f04e984087d095c875b7ae7b5542", - "content": [{"annotations": [], "text": "Hello.", "type": "output_text", "logprobs": []}], + "id": "rs_0207c20ae125feee006ad023abed4087d09d32a32ad2d3182c", + "summary": [], + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COvx3R_lOOaYJijiIokUKBq9tt_p3mhoPHnm4KR0Pj_t0wf1sP7WEDHfYSzylhHilYoKjU4guIz-TAqBV6HNUpIx5ggp3aukKEW6zks2ERT3a40sCFFdUdKHk6py5trFuwYn6lHLwifwDkd7rI7MKIP7C7pZSHnXh8cmfk0t96p4nBplQ93C4m917Kyx6PqvWSxPP78h5s_RMEdhkWRU20xMhminpIBNArOkPYT4xsmBEuONlndM5-DHG7ZeLk2yMZw8b_hxk2LZ4mC66b5RHfJKVKfw4egn4K8tkr8mQSR4bnjpY56DsIRj4fjSH5GpoDZyolFFBh1BNkGMoVsicye_4gZn0eELuOWxCw7lVVsMpIzTLw8CHJkCo_8vKtNCFSZcKjOPKDDfX1bsFJ7LMonmqijHPV_kZi_YXjVNNJBhJLiGjYrVUFZzjS0OKQx-RnZf5luO3UM2Ff3Ilvx1eQtuWlsvJjPBRwObzk5zlPEdYwoh5OjUREbgGb4H71Q7A9YYjUIms2h5e13Z5s8X0onRVr9SUcSbbMA0FXWn6QGQ5qgiQV56xec7xk62SrxKBXeILgBttBf68YfLAbdINCPL89G5Z1X2toQLlFM7e0jKYUOvSmpWyPLZ3q1040oDf2Q92v6NjOwk4WlLtZF_4xrZ7bEIWmea7zcDEnsvNBRdKaUbeUOm0SfRuIH4gQfbbueSqM3i12eK4TQhTbJ9VivUUovUv7pCKvizdQuunf2MMXWhq79wLYQYXIleQ06E9hbx6QT-Ue5ELQVC3NSLhXiwucOSqd-NZFDQA5wgW2Uaq_-bFYzNOPwtArfaqnHOILmr5mDvWCrlnLs-DUPHU-DtzYH7Eu1VSlcEZLaZX4vC9ui3x2aOpTtmVCILZpw7MkouVxSR4rFE48doVoCamJHhey-TZ7OMl0A5JdacTt6oUE8Z2HEcuVh9yGAZe_Q-YB3p4egLbxO8ICWUkwSSNgrPq3WRbT9sbT-Ko7N7KDAfgyvzzdEYMKDmci6GBrVsAiscc97-gxbhVhlrMyIXmgTluW07o591YjlOkvERXeJobEfby-VUJCu0cHd0nMudzrhCpHIhU3aUMkk9GMlSNDPFnKeV-gylycLhYRb0xo1kXMRom82fKmvUgO5Wh_BhsCdeGaVQ2bm5wyLT4x9VhEe-uOwaYG02XxoSiIFqqSrf0J8auFHIRpUTKzNWMK_GcIaU6o3ON0wOLKaga3wcyrGkG6wtvGkNLCrvmDIXg0bmvFGZrmybQvys_u636NXVS-Y9irqUv2ghu5tyKI_F8uwibi9JkKa5XcotQGbAToTE8qaHfowZOdTyjIKrU3BjL_Hs5M41KXwrNkk_X5ttJxEPW1qCjyusFb9yeGa9wzHMsp_VO0ysdO8L2KSRAY4LHpGI175VxUmnSg66PtCPln0OYxWsX2FWE1A7B_la662tO4VoS3bcslVZreRWFDJ_7X8qKx2fStxd77B7JkG1ZElxlZYlBRN5_Lo_w1yM9SvqHzzewQYEprSA1gltjxAbIzy7xXReaT86PNI8ojSLfXOkXa6Cybu3hIxsIomkZwTtAEx85BoIDn2B8oTp744Lwkwn4ualAfV9TebiqOWCUYp47fX9rimte71X6Hnvf7tWEBZMMwSb0pTqY0boMOS0w3XKKyEVRNyCoGWMgyvhAxk9E5iao5N6i5W3aSel27QwsUaE4YDGKKMKSryH3BqP6X7d0kdUcAx-nd5yudVbePQyc-UC7EbISj5QXY8g6cTOy-IkwpBe68DvfcNGIhuaH6UscqrewXQjKZFJC_BJSl-ICOfMw-FCTumzCDXPqTeLBPHf1g5kd5lBd6HLPUldHtJfD48V12KCRtaoksZ4g2NibuClnmZRpnDMPDUr7r_D8-RfoLEiJeh8SbOSnLvMPVAJIB95F73B-UX5u0e0aWeyfG5QjBq-kB7vVGsRiMyHEVQW810xo2qULVIyVO03s7rAWwTxFY-EHmKkaAIbKU2b-qAasytni7f1Aq61OSIYqSCRdFhpUX6HpnQeCFC_s6_Q9P3CFx120ZaTB_HPeRJO6dhVSJ3nL3PiNi8sCnhRiNIXkJfm9l1TxRS8ok9s1NY8oi4Y98J4Ay52EkX6f9F9FtWHNu4GX_Z50jDNnVTVqQo3L3slpwxECsTAEU-0MsmIa-1RqJ00VqBLKtIDVbbnQ==", + }, + { + "id": "msg_0207c20ae125feee006ad023ade23487d093e272f84cb35723", + "content": [ + { + "annotations": [], + "text": "I’ll read `shop/validation.py` first to check how it handles " + "zero, negative, and non-integer quantities.", + "type": "output_text", + "logprobs": [], + } + ], "role": "assistant", "status": "completed", "type": "message", - "phase": "final_answer", - } + "phase": "commentary", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_S4WtfrQwa6ZLEUZf1BVtA8ON", + "name": "read_file", + "type": "function_call", + "id": "fc_0207c20ae125feee006ad023ae893c87d08068561e36e665ad", + "namespace": None, + "status": "completed", + }, ], "parallel_tool_calls": True, "temperature": 1.0, - "tool_choice": "auto", - "tools": [], + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + "output_schema": None, + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + "output_schema": None, + }, + ], "top_p": 0.98, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, - "status": "completed", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "truncation": "disabled", "usage": { - "input_tokens": 19, + "input_tokens": 309, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, @@ -280,17 +267,16 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, - "total_tokens": 25, + "output_tokens": 118, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 53, "text_tokens": None}, + "total_tokens": 427, "cost": None, }, - "user": None, "store": True, "access_programs": None, "background": False, "billing": {"payer": "developer"}, - "completed_at": 1791225934, + "completed_at": 1792025519, "frequency_penalty": 0.0, "max_tool_calls": None, "moderation": None, diff --git a/tests/integration/translation/responses/bases/vertex_ai.py b/tests/integration/translation/responses/bases/vertex_ai.py index 7d4fbe0d804..734ebf13613 100644 --- a/tests/integration/translation/responses/bases/vertex_ai.py +++ b/tests/integration/translation/responses/bases/vertex_ai.py @@ -1,14 +1,18 @@ +import json from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import RESPONSES_REQUEST, RESPONSES_RESPONSE +from integration.translation.templates.vertex_ai import ( + VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, + VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, +) GEMINI_3_5_FLASH_THOUGHT_SIGNATURE: Final = "AY89a1+kriNs9YsPegJQxgjTWsMlxsqPU/UkHDyz4EK+3FRrE52ygqPIHTvkblgKKiO5Fn8R9D3iu//i+YnuuZ+e7W0yxUoFScas89b5rn3vZHsihnUA7VTe3b7beth1IftteoTGZ3F8OSLrSIP8HKn4xTIx35eeNEt5oiD3K5rTQrdklqO277HvTDhu/LuZhZ0pzaWciGQEc61epIBBpC2ygu8V4L8TekUVZnz6qzSzxipdiQStQ8dZou43334hMnxOR6JjJCGxqt/1K+C0vXX6YP6f+Bph5VhPSnvy2Z1uApi3TUxAcsDDyVewIr+ha2G6DPhN1Oi5uZZNhmbvH01Uq5OOquF1uEICq2m/35LhJIbDNM9N7oI1Z6L94P9xdugeFXM2S7wdJfc8sdg09vEzNeA++GK25zRuB4hU7XyaG4XnWjmu2kswi4sf+EyrO/K0P2tb/ZCJu7kJTtWTiD1wgSIl1eqS8ORKKsXDN2hIbFrtsNR50WIvVzd6" -GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = "AY89a1+gmp4bVJhwGOCtbUalmyfhwvgE4EsZOwS4bNR02vurTt4AZC72ypTNLJDRwK8I518tGYBiDnFqD6jTxkjWTQTPGwUZ5IOYr1IskTb3pPKfvZWZ/XunV7kfb9VmEU0TBlgwjVqxXziOxvjediJyb2gbFnCVjGTSVsq0ipmpFznF22lV1I6N9GI7RKBFgYXaSzGU1+fGbLLBigaUD0qbgquAnFv6R3JGpaLcW7EUrhKDinHoFukmUwBLx07S7SfuWBT8hMLMs32vEVzwpns9JmbVtuQG24zs5qQx9sJqtO9E07k6SFL3f1tvaKZsQ6Yjb08Xcuvnh9Pcz4ChCdRJ4vRkmtRW3AgOli8/43rrrAMmvIOqEdAOOvZW3W9D8lnOC+Nfq7S0FTNWPYkIJhrtskWoQkUwDrY6TOzMjIA6Jblt2f9kKU3VJzq7O1hPjbHsJKmLBYP+De8rKx7Ufq9Shs5qLLu4aBBgHleiyFiHGQSD18RDjnyv0DLRyrp7LOkjIYWBYoIA2+BSb0ySMyeLhk8B2U8XZ6AXiw3aHoNUG/wwuE0EhXBhOUBM14FmchGfptXUr+Dn9kZvPAy8mfbq3Pk=" - -GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE: Final = "AY89a198sdR5Cy3qgtvOq/o5xsmH1g3WymzyZyB/bv3XeyqNZAgXbCocCAqUS913U8FvVUodNrm3y2B5P/mDsn3A3pINhbEiwJcWA/o8bNJq+Z/1O2UYdViO0FMhOOS1aRaCbza+6aSBLPVQ81QnyxrwH6rdDCuQlwyZ+5/Evc/EOXLVxdUF9PYwTwNJcZPREl4UMcarKvtKsNofkSfa09EWQNVVMDkasNl3YHtQJqYLu5rL7vSUrbMtLLA0sMEMh2/4zMj4gIvtjoOwiI4CBdLLH0Xp8SGGco8UljxikaIp6YvQW4cHsfkaqTBQmLgREudbnKKTZ0MvMR52G/++7r8Iv6BL29+73igMMyHoQad40OM8RY3E6akpSE5uPXH/CgYTmSLXQONtRMXzrXm996JYLaHLOo2dP1RtaJANcjNoDB4eN5vj4GOpTUYqZqRRaB5jMX2od8QcAnaXk2AYs/2rNMaLV4zgraz45gbA0It465nN5hb4JbRH2toJ9K/TvsZY/YyoLmV9RjyAmIhbq6bG0fGatsXgmDaXj/vEsBmqfRyBN31k" - CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", @@ -387,16 +391,233 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) -CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( +GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", litellm_request={ - "model": "vertex_ai/claude-sonnet-5-5@default", - "max_output_tokens": 64, + "model": "vertex_ai/gemini-3.5-flash", + "max_output_tokens": 1024, "instructions": "You are a terse assistant.", "input": "Say hello.", "cache": {"no-cache": True}, }, + expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.5-flash:generateContent", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, + expected_provider_request={ + "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], + "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, + "generationConfig": {"max_output_tokens": 1024}, + }, + mock_provider_response={ + "candidates": [ + { + "content": { + "role": "model", + "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_5_FLASH_THOUGHT_SIGNATURE}], + }, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 9, + "candidatesTokenCount": 2, + "totalTokenCount": 95, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], + "thoughtsTokenCount": 84, + }, + "modelVersion": "gemini-3.5-flash", + "createTime": "2026-10-05T22:54:50.809002Z", + "responseId": "uirEaqqwMc-U9LsPjJTYuQU", + }, + expected_litellm_status_code=200, + expected_litellm_response={ + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "vertex_ai/gemini-3.5-flash", + "object": "response", + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "phase": None, + } + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": 1024, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 9, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 9, + "video_tokens": None, + }, + "output_tokens": 86, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 84, "text_tokens": 2}, + "total_tokens": 95, + "cost": None, + }, + "user": None, + "store": None, + "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, + }, +) + +CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_0: Final = ( + "CAQSqgcKEggSEAIYAjgBQgh0aGlua2luZxIMHBaYAp8JrEKnRBnTGgwijBpg0NpoxIwa1N4iMPW6lp+ZNZcZC05RTtExmUy/5WyQ" + "mqBpk9fNiE9PKmQSIWh/l9NOxKOPAqBJ3uWJQSrFBjHaiQlVkJyGooin2mlHXfqZeGtr6F/3yL7fhsBTnYiZlZfLa9GwmzIZqPY4" + "YhrWpo3hLZA9+LdoJ3sOeow336Voe/8M/eGeZDyVV33rgVVS1CVAanKfGn5vekIpLgR3EesIdw9Jx/NPOv1CQ8HMAnNZpHQYYt0P" + "5uwr+BVS3GwAl1Rr7qlaFHj98JB6Stz+qOVGGJsLCY3JRX4j1D6rY3YQUErP1NUpUz/CD1ugOCkEzDY0/EHGaqTqenWq93YLtsdk" + "9qqovMFXBPzsi7JhWFmlzqDoNc7MWGQHVsXvGwN678l9r28dD2A20NUJWWLPqa27KEwEihsBkXIEqFDAKP5e6tdaQWzuHqrPws3/" + "M/L8+mf5RzO/ThtJZucdqMBAxC3HKqutVKArQym4dyqDj+V+vborLKcPNf1dEpY2aiGCDPrDhp5r+p0Fz/4++9V+vdZIy8Irun4l" + "ILkAJOmjgmu8B+B+inbR/bn1sd0GYE752RqIWUYk9o1QzHfQ2XuiFZy4MZ3KaJHZr3eopTaPrdigJey2s369t5ACnIbEjFKINjgz" + "eMfEHMrn95VaViTOIJJyYSN+GDEmcDobpN5OXrQtk5xSeF2CZ3uGOsUNESL76sFqt7pN69uxSGrLHohdQdQP3BLQ78ss6VcbskyR" + "71/ZosbZDxA/vHIh59vFu7tR9V4E4C2xkVUHa5jndYCO/KIhp/Ce2HGU7PIGarsxw2MEFt3+eh8XrS6h2fbQvhB3MDquv1sP6sVb" + "SZJ0tCqLWbpBO2Iq+3RAroS0+96eLiX3KKwuX1HVSVCgU0xzM2wobAh2NIGWuI5Ip262NArowCEAKhTDuc2zYle5pjyQ5s30Cmr/" + "xcIXxEPcmBhIHxEWq2/Jt2ejbjLgzSZiA3SLU3SgTjSI0syfWJUMMl/9rKSyXV8hv36rk6zu4U1nnBrlOHuXKQ4+kVgfNylFaGRb" + "g0XbHas+1nUGtH7RHbnFDJJBwzVRIlyq+wuPN4NqxzJYwrjDp1AqsFo3VJvbOx9u02lXHXRk4IovIQ2o2oA+HW9ZN3b1wDfewWnm" + "aSu1wmkYteDmWQntg+hCxhRvAjWwdWR6nE9G5MxjPWRIRmt5OP3NFtBJFBgB" +) + +CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_1: Final = ( + "CAQSjQYKEwgSEAIYAjgBQgluYXJyYXRpb24SDNxOsiEvvTpgXLFh7BoMc2y30nGjlC6Jr/BeIjDZghnhINTjICZv6Mz/M2Vs53Cg" + "ukg2cKEsEdShw2izKv7NanGF/Nl2ua9qsvKyxW0qpwXQ0xd5Tr4TUPYyZAy3GVQasB504SWCzSbA26LwrWtoQ1M9klTwLoAzj9S+" + "h25IG2wbG7JbhCHgRUDQ+HlnwLRmkHisxfrWprRmXqms6EOhoU7gUM7mGtPR9Ttxln1xkZWZRA7V1N7RUb1rJA7itM4zX7nFwk3Q" + "G4r9zkyd1BhiDV20j+u5+vEoNyeqT6XB95lgB2zvNZGUbtqdYlagOBS/bZWYQgzOGIsXKid6fbCDA0Na2EzNcG9T6fqx+EDBEGIZ" + "JMP/zDxXVjXH/wUUi5YjZfGNPTncg6x/TT06RqRVn8phMU1l31sh+xcz6PS4tm9k4ysckmo/SxPsaFDAdgqepldXCs6Tl7viY/6Z" + "abgJgkxtijBp9BsslKJO8hbualPuhHuBZkYJQk0HXm+4OyL4MNyswyxLP813NILt3L5txnxpjFbHWenn1hiwvcbYEUyWogxD79hD" + "X5MZ3f86B92vk/BMGRM9TUSjphHO73X00hL4ZOCwqghvjvorRyyj5tjVnbFrAFCayaPF/QAwBIuqGFgNOG9wMF2USzioTCN1LHGM" + "d7uvg5tAmU6k2/p1Ea4tmsVZF3h5rAw/bAUsF/msqOlw16NSakaGzpYYs++UFF4gYcR4rpROqg94/8z+up+P0Z2vrVVbn0Na7ObD" + "RB7gwr7x1M9XQKXMp3Jtkj4V6zmAtci2PBjpIAX13y6zpR6xyVQECefg1bZmTjWPo2Ev9XNdXC8UlhersNTg9dTcnk6I1jfofBVQ" + "sOPu5+mFG3rsZQCs3vKv5IS3j8zBFu3z95o8RNKURECQ5VcH8a1css/lNqhHhUdlRhqx6MlI1ZUd8eyI1hjjSzy3gn7VuZyOgAqM" + "oHonAlebFZ8ARFGYGnWpid4+QyiYUyv5W7EnoOD5OpQoq2YOGAE=" +) + +CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_0: Final = ( + "CAQSiAUKEggSEAIYAjgBQgh0aGlua2luZxIM5V4+5UUDIuxI11UEGgyO9SbnX7cSxGWVS+IiMD/+JMqCQUMJv+EWD6oB6YZwwIG6" + "O5aIuop3rY1uYL76W3SEEhu5PI8DTabb70ki6yqjBLQCRri5xHeVVrEpRiH91RBZghREaOtv/TE8aiJYQ8njspRXIUbTj7PGxmgQ" + "6xZ7bAK1WyljtD0zG3VTxFx1dSw5a/25/WJuYz/6D8gMXPQhG0j8lZzKfIvxhOq2mB5KBmnB+2agzXsyyws8nQ+tT7s/Yfyi7SbY" + "dH+jysWNoKj/0pLiuGQE8gmmhkYfHdyOl518sVn/C/yJ6mB3i2Klkg8WifS0l1crRejKy3Er1nD2HBvwVpoTiXAe1v5ugB9NmSve" + "pgg/5E6sewMCAnGLvL0Z9lfNPxpRTp9n0yP61Q49Z4kvSB9lcbZC5wN5/fjZTMNRQNDAEqhtXkWnA5SSW3N9r4Pcls67XcYGLI6f" + "kjmJU6GOXiRbFIbptxjv/UBHmXPNBqqnIRioPi2RE8eGBbsNIjYrxBD/oJHb8Zd2RYeaM19UtkkYoOh/NaX9iZ0Nek2rYf55YOlo" + "GowYdz6aYoo6E2xPQOFh8HAqF0Jzj6KCYodjidlI5jQHZlaE7901Q/u9y7AP0aQXrfAgeVxEPdPUfnp1ROVzK7A6K5O63Ozh+3jB" + "KeZRW44jj5vA/+H2+uEVyv1x3eED2eK9j1wcCw2vIE3vTDvPbh3g7RkraTcrkdkuRHu2KS4kkQWppEeHqiJEmEFwwbCFHj41dsj1" + "G1pWc9elR+oogW3sYKHQriwykjTCD5IaZi0H+tb8cNegqngv6kGWF10HCSM/4yaFqeATeWEYAQ==" +) + +CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_1: Final = ( + "CAQS1AUKEwgSEAIYAjgBQgluYXJyYXRpb24SDE/QqOK1lGVBzGebvRoMg39CM/sTiL2sWeIEIjDK82bOn+vjF6wbbaDm2uaqcCAx" + "GlVUbC3CIx+rc3Wzsy7+0FhVIMf0D1k4GYnCN5Iq7gREgQE5RcQoQ7LQ8HO2ato0KeXeJosd/8ChrqGphPMNQPepQDjydUBgG8Z9" + "BdfyxAaoAjxJwSgPe515bk9xajqoTBOlQWELydQyoSo93+SCnHQcCquftqsHbx05i1mgJZTm4L/8ON0jodnY6+Pg7yHW2QixEAht" + "fpaLYEH0+a9/rYEiScsBls7gGPxcm4PuXJNt0+ymg3JmbcQ0KQ1sWW0FoLbfVSr0hQV64TahV6FfOVjug/hht/EyPJcSscrC6tbc" + "e6VuW/n/LCRub62yHgJHQ0bqiDGi+uQgyhe5UgeugSFoSzmgLq8yf8TeKyA9+Pgy2VHFN+BOlW+M3s/9UFze0BBKgESOS9AxxwEF" + "AEFWMG7UxksDvsbJ9s+rlXDpR6Zm8SfvfvQR60BAWQjyntxSd+F5RsmaFb369VRkVxUtCU6zrZpdkmAqmjHTgdBksBIEz+u/otTa" + "TcdDNPsPVCqf+wSpTqY56X5eWWDXXimQGGzKZM7ELLHr7tvTDKfBmRsOqTisRf7EBnxRgD3IAw+nDaVweu+hVOiMdF/vLlc9MOSg" + "75fnDILOJcmd/QiCrB2gAoUFzqLpNkyLUPOiqWaDkyWgpeD/34/hKZ0OmPZDuF55pEWQ0YwBxcw9YOx0yT5vqOhp+g4DP2dm4OUu" + "ndhJRSGLBFJSCVdWdezujMQKacV+fOlidp7s797i7efNUzMyZqg056dN9l29fcU/0GKrpBXv7+Ne3vl2RE+4TcPuN9i2SWDsku9N" + "yfwIS2Qy/LluH9ROHmoFTv5CSKxmskP2265nOWrdWbv4YUedA7spiQG/OxMuqF9NtriPexQeGAE=" +) + +CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_0: Final = ( + "CAQS/AgKEggSEAIYAjgBQgh0aGlua2luZxIM2adaKStJUdqWjb0JGgzx6a0ntxWBsWf5SPoiMJUxhiE/xFsZl3STVS3np3WJ/BKT" + "beLsjANHlerzAYMPNBVAhHn1udBy1TF2WKg5HCqXCPpPMw5IgVP28vObMyGSjS/x+oIuZ28MO+NKO2I9UT9jjmIpo09b4xA1gVML" + "V74CHnfCfqAiSyVxuu5tltOyp1Jn8ctKef4JPBV4KRYl7wRV5MUKjicadiTPfsJrSDigGXTlNBnP4kxP7FHx3a39sBMOZaeqcM+6" + "PeiXYCkNQdFWMIlRkELFdSTwxiBJLzzv1cCOQWKic4RxLuL/9vys/Yi+lNFHweovcRg7YbQQ5SisK4wSUoaFvTfOQbvHeN+THhfh" + "8YCJFz0eckG2tAJ641KFkuanRvvvTrAW8d+ObpoHjtFpqF8t2NGqmKLnQoWxoYixWS+KPOnOKNs2qM1LGI2Ng37JlQGRKhHEmdHb" + "uTv8MQCZYAeyp+6VQM5cP2CS7qsae8aLJ4uLAkwack/chb/QxLtpmkfwePDcPhmYC+mIpaWojs/M2XfbJmPTa4w2NklBELbpvTue" + "QRgGhOxBkY4vhWu73zeJ6/NM2CoiCpoLJnty5eM//6/IA95o4cIzNY/3WPblFKh+fVV+Ilsdav3KvLyBGnmt7EcM6VTmLmi2Idvr" + "VuHaaFNB0PRSNPP6TXlEd0AKOBkCLZpKd0wW5vRCo/tgbj7Hat3ZC8ZHGMJhnGGr714wWH9Wog7qZkj+F9C5LlythT24fMt+C4Fl" + "lK76u5XMxI9tbfirIq0sMkzqEaJk71QqkCvr6gs8lSJyXhaWC17X8enFEIxcMFlX6NDBsLYs5YVUUqlNq8Wh1JuX/irKmm+T2OIh" + "+7WHUnmTKAgQpEOIuzA5ia95dLxRWP/jCUvwDRbnvug+qo1iQBz3cOkpKUjtbIEx1ory/wm3wEf61zaszTOJDy9wWe/BqpyS5nJO" + "g3ZEKgy7Yjle2J411sQc1iyZ3dVuCrGLsZoa4ANoBF6SFcm8evZcq9AWWExIG2mQtX3y2SzWyIIw18/w2iTYTniO8LFkzNZlokDj" + "e6SV/6d9Zpcvm75FtQurYETKQd2WPJ6jnYZKPR+DYVEro/urBAYtL/2fII5pFNi8RMJaXOsYqmHllDWY2a6fX6KzlIjod7XDBatD" + "IwkUkQNUsz8Tmdo/OLY1kcC+bWjPcXzod/Spxc4MUeung8UpF6WUm6yLyTAbkdieIaTDlXg3SlEFWz55qHhskzHmi6i0vOCly7Dc" + "nngc1swUh337xmnQIKZnrKYom9X4UiItcex4cQ+C8oKjmHdZT3l3zA6b7m8QZZpd3V6JQQC0IDOHxbvtuO/OC14OCjzeDzBKnJmv" + "qJIgA7JG3hjb2eDPDGp/THA6XDpKFOHjtRX/dajRL5NpSw4NFxrCOCN9UTXmzFqC7rwM7IiK1Eu1VYPh6gXKlDwqCP6hDu+zEajU" + "HOu0HiQvlm3OECugNo2mtm5gbrtrYq+LYZ8MjxgB" +) + +CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_1: Final = ( + "CAQS5gUKEwgSEAIYAjgBQgluYXJyYXRpb24SDKtQ5HPiP+fYCKzUGxoMyrwawBjxXUkBTvgEIjCDIS/QuqY9Vm/JNNkKKFxwTX58" + "kwQICd+wgim/pJgr2sDZRzdBVklH+aNI+pFM37UqgAXXpvcziXDsrtYuOqyrTaXNs0t7+LUPjZMRiY4pfcVvJbdJTq0R4HF1VfsY" + "GIOJuJLKw2uu8JM3t3+wILwrnaLBjs/nHojjdOuCCLEzatKp430zN7+slVpY9GmEUSnJunVI38Xye9f3YA3jAl/yEtrXab4nFn+w" + "KOx9uTlHg3s7TZC5W6h8H4dfb71fnWFU5q0hBGw67aKa16tr3xU6/J66gOlFnah7LygzYwn2Dv9O34Rf1WMzIXZgf0vv+BWtCNP+" + "u3hOtSlAprySuawJx6T0hSXqDIcRjC3VQqnDxw1M0xEQSegpFiBEZsrxcS6xg2AEpIJZaortESXGlKDv4WSZ/WvwgcKlZY1IttNd" + "TDlYvRwlKiHv7R2KVGzpyCx5avIVV/I+QLAsiWfoxwhauiFSnviTngaoOujAYrKe/KTZi3Orx0PIKDga5829zgwSwRkJn59iLcpr" + "D4clFG2xBRSnjxa/g806JHH0uK/Ko/wJ190f4oV0CSBlv2A/4Gd+7OwwTDmXpc+Q9uhhcam/isnJOo93Y2qNCYYBLkv0nHaXry9G" + "UD0yqRT3swkDdxrX/08DprwAtht01SmPl/yRVG26stOuNRXhPFqat5AmoiWQbMG8fYll3/lcqn2LBxnQZFTN8LVZ/Hu3NiEi11Ui" + "Rq7O4owEeyvdfh5YAjc50bhzTWhi37W9Y3YcbuAcOsMlOSLV9dwnTz2WGfSPDBXuFV5ltbbN4tHfWG1l2zs8+MQ7FGBHQgO6aLP8" + "ZbNJW02ePpv0OOQnbw3DOw83MZahNfOABCQvwSsW6//YPhbSu+fZFcIwhFf13ydB1ZRu2xY/9wpfQTRKydDUEw0DyGOX8aC/GAE=" +) + +GEMINI_3_8_FLASH_RESPONSES_THOUGHT_SIGNATURE_0: Final = ( + "AY89a18zqWYsP1fCe9VRYni0L9f03IJNDHvm1boNA4Q2dVNJPzZZvrxcHGuC7kCwx5e9kBdKsGEQ9/Nr4AjO5c6ukC2h2NEeTinm" + "hak8IRJNBkDQ7OZ7at4q5KYoqYfDZ8aNohtXWBxVdoSK9YZHp+nPGYE7zmXk1icksF+WJG0ga0jBaivzjYeChB7DEp/pEfxa5e/5" + "etULNzsfurb4+l7Aht/DYaH/Z+EEttmrxrDlKI9SHGOfDy8YRn7XcDJyjYjIJ/Hgcoa3/STN5kaEJs+7FzyrN64sSOyhzsP1H5dS" + "M27anFBLUjs5KHg15CoDirSbIlRLlOHJWQ2k9Ql3ljBfNePDAds4/zH2VchSqiZfsIS5XwUGm8D7L565vzTzknZtb+zSZivDPPs/" + "fCe3iCNYY4YEeSt2Zz/hy7baM06u6Vb1cljWtmckLoQgpiHJhY/yA5IhrHLqTx+PnBTgud6+yx7r/9Wiu8ttxeU8vupqAERoBj9y" + "OmysvjloE6N7g/9qwsRvvoxICo8luVQmA1xWqkDMTLT3LNfEwIpWmEr0Tx+y4chAZitmuKUBZdtPfrayO96GB7iPjRUarEKs/px3" + "hHyaSs7QzZZM/ymeeihqYyWzr81GN14/MsHs72eX3sl+ooYhxM/qhpOSjFiMgffLrD7NlAwcBtXfNI7fQWjbkLYf+mu6hCMSY8gs" + "hIkSLFcRv8ijH3eK2RBQWzvDsjkNgiuTlXetgYFVe5/Q3YJvCCdPsbn/Fd/dc3etq5XTGJ9B1ecsI7KFdWOX8Q5HrrfWFOuo7BuW" + "XNnFBN7pd1RwzWI3cveysx77EgQ/fMS0Iqb9FsM4i8SIY6tW5KIj+Q0SdnGR2Fkh214QxDUwT53ThPDes9mapBpR4lxIqvROoMcM" + "PoMFlcvENF9S8oeE17cifQTVFrQT6xGeWL3vV7AVGVhuP7RjIX9cU/NK2qBof50qfI2BC+aqiu6WtzWQ2IdfrhCfCChccr0Jh4Il" + "dyF/0mW45PijT0mguv0vwceuVrtwLkYSK7gsnb6REGUmQ4WXWy9VwrCmcYzmX5igAN/ml0sML0+k1pDHtYVcflnjAT1s/wiQFJ4P" + "g1vUXmpjGSOUb7KiyKeiUvgTTdqD1EXHAWxnFJtgwwz/6p+VhkiZvvwOJY/GSciX14pMtA57VsGd+2eUTRdpuEbffVWjuuyeyZd6" + "Xgm2PNLnCIvP8sS0FLJAKhOYau1OmGwkl5gsteZqSgjg2l7hXJf3lHhXv4UrRRj/YwX8SEg/YPSzhurb3bBdLKBUPfQs3h/DlOnG" + "bQqi7efhq5seYJBldPpiEtwChm7o6mJ6J4cr1AqLTtDmQR6XF5ZN/oE6XGHpa2XY9N4tJmSrnIq6kuunngTfpD7yejAg9bHVy8r4" + "FWjU9wU2AgYHBZneDXTiVKDaJDzYPdoufiADokGyWntWksp+B3jH+El+kI7c3uu/tL28HVep3u8Jz/XlLqEgh4oViCrcsyqqjLsU" + "rfQ8oDmiDJZfbxECR7GHDxJ9yA==" +) + +GEMINI_3_1_PRO_PREVIEW_RESPONSES_THOUGHT_SIGNATURE_0: Final = ( + "AY89a18mH4gPpBuF8sZq3BxEt7fVfeky9tDk6WrFiAYqUNfy1TwKL5UEuc60FQFzZzoY6BhtmNYM/YnvWntgJG06tpD+R69L+px8" + "wJWN2jf96nYOxDNe5xM3Z6yz7QO1XikWun/bqStP3GVvN2xCTRBeQxkN0n5+NPDWYXC94zo2uikRYf1XZ5IsCrbwhDgFhqzyv+3C" + "pEmBpN7ghzonZp6GL23N0ELNi95QEzZMcGrD31Z3xx6Db+KblQL9yVKVj7STHJ4Bno3sU6heJx9NMJbM55wI5SNcZqi8YqkS3Kli" + "WRxcPdWbkekAOCWUFYYFG0JEB0iOibkqcCV6j7bZaJD3v9qdqpaSbG1Q3yl5Ncb1KMMGLwbMyjN9t3XrFQuqtM1x3OE8hofM3bNP" + "MYGna3DDEmlporSsQlE6W8Cxa70IsBaFY5h7eQjKCkxLSOhZmGfmuukLLpxXWG8ZHzdtOB7vmpIcBolW2TcQCamojGpvWdxKz/Jh" + "U9WB9xac6A5luWXnIA+YC/Pu793XzkDrST8jT4bei8Wi37sfAhRUkVeUw6VAAYbZKONNM43U/ih3BwJvpJoY5voZVyhm4JTT66q9" + "vSTUSiR4oQz/ipXDUKVfWFhqEWoh3Nh5elhFe1oZyx8q+lGwXm4VcrGNtb5xwWGnAsQN9GV30egFGNcEKFq3rnODehteSTTaHF6Y" + "aqklMypgS/K09SfshF0waKvX0GDouVMbnO3bFfMxGU1CofWB3b3q6RbIdv+O37WekKo0O8S6OwQRdpfOCNXcwpwsRczX4s9IoGRK" + "5IIKr1/6cQze7LA2Gi00gBIe9RpbSMJHo/XJyDsP6KT1IAAtoEkyGBzsVjBEU4ggVsG6n8ka5y4myFgwMQeNrN5DNqFL+nVzxqUF" + "97pRU7Xw37SlNNi9DgGnqhyWY/T6Vvr4yhCsYR2BElGCq0C1YUF9OQ8+gp2CBrtd+bRLHz5iNyUMoPumSAn6B1+GyU5JFPQwC9Yv" + "NK6ug+9s2akQgYYgNAnDS7aXj2djQ57nwPqnsg==" +) + +GEMINI_4_ARGON_RESPONSES_THOUGHT_SIGNATURE_0: Final = ( + "AY89a180ZmYN6Zq/ICj0BCu+eBS75TWGlATeX2fsIhoKaFuGLfTjmizARO/YOhSzt4TK8d1jl7skKbDq23TdvRVOJfjPsI85UiVC" + "F8DZC4gSdHQbTXiDtBUsue7BxMBnoNfPJe34Y1p6WqjbaK44zsfJybZVsWJSmQOW5Ky8qnBZXfEL9dyz2j0rQbN5IMGUV36wrA4v" + "KLfUj6vS1dwzz2LETuK1rT/EFKXDG2Pwgo11yMzoZuPvC8VvNrQXqJ/Tizd/cuuc+Fiht7V4EvrEDnDH0ZFMW4gyS52mfjE8/hnI" + "FIgzaFkhP7mWHBOwEt3ZSPZPQ2d6ZEOXD+1clQXSOWMv9Ks4Iwvy8hrcri60cdqbFdbHxF48OgfWUy2dUycP6kuKwWUJ+sDNT1pN" + "lb6pfhKdmP7wd6QvjudKrrunGZbWr9CjkiAGVOD8RfcQK8Z7LUC11RyHqGDXLxJA1kzR4W4wct1Y6uWF3TRN8QsSLpuXLS+sITYs" + "vEaRtT/wcCBVYu+dHephEAo2UXmaMfkZ8QxrWtf/VOwDZegQiVpBkxOZyAoByiDJrCwFa/osvXXcj0BtweIKpKKUdLy77BVE7IHa" + "JikXSIRmpwYZRnA=" +) + + +CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "vertex_ai/claude-sonnet-5-5@default"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-sonnet-5-5@default:rawPredict", expected_provider_headers={ "authorization": "Bearer scripted-token", @@ -405,93 +626,174 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( "x-api-key": "scripted-token", }, expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "anthropic_version": "vertex-2023-10-16", - "system": [{"type": "text", "text": "You are a terse assistant."}], + **VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + "system": [ + { + "type": "text", + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the responses " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], }, mock_provider_response={ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-sonnet-5-5", - "id": "msg_vrtx_011CfjuywExcu3BdZ3x1KzVJ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_vrtx_011Cg38ropcTjx8YNUg3LxXa", + "content": [ + { + "type": "thinking", + "thinking": ( + "Since the validation and tests files are independent of each other, I can go ahead and read both in " + "the same call rather than waiting sequentially—that still satisfies the request to look at " + "validation first while being efficient.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'm reading `shop/validation.py` and `tests/test_validation.py` together to see how quantity " + "validation handles zero, negative, and non-integer values, and what the tests expect.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_014c2eZQXGiRtzh9KvDXRhRh", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01VoiC4J2kb7yivWHYGqvRRm", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 50}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 414, + "output_tokens_details": {"thinking_tokens": 218}, }, }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "vertex_ai/claude-sonnet-5-5@default", - "object": "response", "output": [ { - "type": "message", + "type": "reasoning", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": ( + "Since the validation and tests files are independent of each other, I can go ahead and read both in " + "the same call rather than waiting sequentially—that still satisfies the request to look at " + "validation first while being efficient.\n\nI'm reading `shop/validation.py` and " + "`tests/test_validation.py` together to see how quantity validation handles zero, negative, and " + "non-integer values, and what the tests expect.\n\n" + ), + "annotations": [], + }, + ], "phase": None, - } + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": ( + "Since the validation and tests files are independent of each other, I can go ahead and read both in " + "the same call rather than waiting sequentially—that still satisfies the request to look at " + "validation first while being efficient.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'm reading `shop/validation.py` and `tests/test_validation.py` together to see how quantity " + "validation handles zero, negative, and non-integer values, and what the tests expect.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_vrtx_014c2eZQXGiRtzh9KvDXRhRh", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_vrtx_014c2eZQXGiRtzh9KvDXRhRh", + "namespace": None, + "status": "completed", + }, + { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 50}', + "call_id": "toolu_vrtx_01VoiC4J2kb7yivWHYGqvRRm", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_vrtx_01VoiC4J2kb7yivWHYGqvRRm", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 414, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 218, "text_tokens": 196}, + "total_tokens": 1291, "cost": None, }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "Since the validation and tests files are independent of each other, I can go ahead and read both in " + "the same call rather than waiting sequentially—that still satisfies the request to look at " + "validation first while being efficient.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'm reading `shop/validation.py` and `tests/test_validation.py` together to see how quantity " + "validation handles zero, negative, and non-integer values, and what the tests expect.\n\n" + ), + "signature": CLAUDE_SONNET_5_5_RESPONSES_THINKING_SIGNATURE_1, + }, + ], + }, }, ) + CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/claude-opus-5-5@default", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "vertex_ai/claude-opus-5-5@default"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-opus-5-5@default:rawPredict", expected_provider_headers={ "authorization": "Bearer scripted-token", @@ -500,356 +802,146 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "x-api-key": "scripted-token", }, expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "anthropic_version": "vertex-2023-10-16", - "system": [{"type": "text", "text": "You are a terse assistant."}], + **VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + "system": [ + { + "type": "text", + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the responses " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], }, mock_provider_response={ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-opus-5-5", - "id": "msg_vrtx_011Cfjuz6qugj8FbC3fcXdun", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_vrtx_011Cg38rY7YVYxBPGMbH9jaS", + "content": [ + { + "type": "thinking", + "thinking": ( + "The user wants me to read the validation file first, so I'll open validation.py and focus around " + "line 18 to see what's there.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `validate_quantity` in `shop/validation.py` around line 18 to see how it handles zero, " + "negative, and non-integer inputs before looking at the tests.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01GFi5nAbXNTTfSxopXwNNPp", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 45}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 224, + "output_tokens_details": {"thinking_tokens": 126}, }, }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "vertex_ai/claude-opus-5-5@default", - "object": "response", "output": [ { - "type": "message", + "type": "reasoning", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": ( + "The user wants me to read the validation file first, so I'll open validation.py and focus around " + "line 18 to see what's there.\n\nI'll check `validate_quantity` in `shop/validation.py` around line 18 " + "to see how it handles zero, negative, and non-integer inputs before looking at the tests.\n\n" + ), + "annotations": [], + }, + ], "phase": None, - } + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": ( + "The user wants me to read the validation file first, so I'll open validation.py and focus around " + "line 18 to see what's there.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `validate_quantity` in `shop/validation.py` around line 18 to see how it handles zero, " + "negative, and non-integer inputs before looking at the tests.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 45}', + "call_id": "toolu_vrtx_01GFi5nAbXNTTfSxopXwNNPp", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_vrtx_01GFi5nAbXNTTfSxopXwNNPp", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 224, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 126, "text_tokens": 98}, + "total_tokens": 1101, "cost": None, }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, -) - -GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/gemini-3.5-flash", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.5-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_5_FLASH_THOUGHT_SIGNATURE}], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "The user wants me to read the validation file first, so I'll open validation.py and focus around " + "line 18 to see what's there.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_0, }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 95, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 84, - }, - "modelVersion": "gemini-3.5-flash", - "createTime": "2026-10-05T22:54:50.809002Z", - "responseId": "uirEaqqwMc-U9LsPjJTYuQU", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "vertex_ai/gemini-3.5-flash", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 9, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 9, - "video_tokens": None, - }, - "output_tokens": 86, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 84, "text_tokens": 2}, - "total_tokens": 95, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, - }, -) - -GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/gemini-3.8-flash", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.8-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], + { + "type": "thinking", + "thinking": ( + "I'll check `validate_quantity` in `shop/validation.py` around line 18 to see how it handles zero, " + "negative, and non-integer inputs before looking at the tests.\n\n" + ), + "signature": CLAUDE_OPUS_5_5_RESPONSES_THINKING_SIGNATURE_1, }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 118, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 107, + ], }, - "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-05T22:55:12.739855Z", - "responseId": "0CrEao-ULbuU9LsPk-mQ2A8", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "vertex_ai/gemini-3.8-flash", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 9, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 9, - "video_tokens": None, - }, - "output_tokens": 109, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 107, "text_tokens": 2}, - "total_tokens": 118, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, - }, -) - -GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/gemini-3.1-pro-preview", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.1-pro-preview:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 1, - "totalTokenCount": 99, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 1}], - "thoughtsTokenCount": 89, - }, - "modelVersion": "gemini-3.1-pro-preview", - "createTime": "2026-10-05T22:55:14.874538Z", - "responseId": "0irEaqqwNbCdq8YP3L2mqQ4", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "vertex_ai/gemini-3.1-pro-preview", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 9, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 9, - "video_tokens": None, - }, - "output_tokens": 90, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 89, "text_tokens": 1}, - "total_tokens": 99, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, }, ) @@ -857,13 +949,7 @@ GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/claude-fable-5-1@default", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "vertex_ai/claude-fable-5-1@default"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/anthropic/models/claude-fable-5-1@default:rawPredict", expected_provider_headers={ "authorization": "Bearer scripted-token", @@ -872,355 +958,748 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "x-api-key": "scripted-token", }, expected_provider_request={ - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "anthropic_version": "vertex-2023-10-16", - "system": [{"type": "text", "text": "You are a terse assistant."}], + **VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST, + "system": [ + { + "type": "text", + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the responses " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], }, mock_provider_response={ + **VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-fable-5-1", - "id": "msg_vrtx_011Cg34Avf36f3MyK2PbrYyq", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_vrtx_011Cg38r5dMDQufLUFGVTBXP", + "content": [ + { + "type": "thinking", + "thinking": ( + "The user asked to read the validation file first, so even though the files are independent, I'll " + "respect that ordering and read validation.py before anything else.\n\n" + ), + "signature": CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `validate_quantity` in `shop/validation.py` for how it handles zero/negative values, " + "along with the corresponding test in `tests/test_validation.py`.\n\n" + ), + "signature": CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01WCXdQndfe72gQeCatYYmiP", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01MDyzK8nTHNuCpY7jS6YJ2Y", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 461, + "output_tokens_details": {"thinking_tokens": 265}, }, }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "vertex_ai/claude-fable-5-1@default", - "object": "response", "output": [ { - "type": "message", + "type": "reasoning", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello!", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": ( + "The user asked to read the validation file first, so even though the files are independent, I'll " + "respect that ordering and read validation.py before anything else.\n\nI'll check `validate_quantity` " + "in `shop/validation.py` for how it handles zero/negative values, along with the corresponding test " + "in `tests/test_validation.py`.\n\n" + ), + "annotations": [], + }, + ], "phase": None, - } + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": ( + "The user asked to read the validation file first, so even though the files are independent, I'll " + "respect that ordering and read validation.py before anything else.\n\n" + ), + "signature": CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `validate_quantity` in `shop/validation.py` for how it handles zero/negative values, " + "along with the corresponding test in `tests/test_validation.py`.\n\n" + ), + "signature": CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_vrtx_01WCXdQndfe72gQeCatYYmiP", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_vrtx_01WCXdQndfe72gQeCatYYmiP", + "namespace": None, + "status": "completed", + }, + { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 30}', + "call_id": "toolu_vrtx_01MDyzK8nTHNuCpY7jS6YJ2Y", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_vrtx_01MDyzK8nTHNuCpY7jS6YJ2Y", + "namespace": None, + "status": "completed", + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 23, + "input_tokens": 877, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 23, + "text_tokens": 4, "video_tokens": None, - "cache_write_tokens": 0, + "cache_write_tokens": 873, }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, + "output_tokens": 461, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 265, "text_tokens": 196}, + "total_tokens": 1338, "cost": None, }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": ( + "The user asked to read the validation file first, so even though the files are independent, I'll " + "respect that ordering and read validation.py before anything else.\n\n" + ), + "signature": CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `validate_quantity` in `shop/validation.py` for how it handles zero/negative values, " + "along with the corresponding test in `tests/test_validation.py`.\n\n" + ), + "signature": CLAUDE_FABLE_5_1_RESPONSES_THINKING_SIGNATURE_1, + }, + ], + }, }, ) -GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( - scenario="basic", - litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/gemini-3.5-flash", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, - expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.5-flash:generateContent", - expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, - expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, - }, - mock_provider_response={ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_5_FLASH_THOUGHT_SIGNATURE}], - }, - "finishReason": "STOP", - } - ], - "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 95, - "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 84, - }, - "modelVersion": "gemini-3.5-flash", - "createTime": "2026-10-05T22:54:50.809002Z", - "responseId": "uirEaqqwMc-U9LsPjJTYuQU", - }, - expected_litellm_status_code=200, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "vertex_ai/gemini-3.5-flash", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 9, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 9, - "video_tokens": None, - }, - "output_tokens": 86, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 84, "text_tokens": 2}, - "total_tokens": 95, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, - }, -) GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/gemini-3.8-flash", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "vertex_ai/gemini-3.8-flash"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.8-flash:generateContent", expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, + **VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the responses " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + }, }, mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, "candidates": [ { "content": { "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE}], + "parts": [ + { + "text": ( + "**My Thought Process on Inspecting `shop/validation.py`**\n\nAlright, my primary objective right now " + "is to understand how the system handles invalid quantity inputs, specifically zero, negative, and " + "potentially non-integer values. This is crucial for robust error handling and ensuring data " + "integrity within the `shop` module.\n\nTo achieve this, I need to examine the validation logic " + "directly. My instinct is that the `shop/validation.py` file will contain the relevant functions for " + "this purpose. I'm looking for something like a `validate_quantity` function, which I anticipate " + "will be located around line 18, as per my internal knowledge base or a quick scan.\n\nTherefore, the " + "next logical step is to access the content of `shop/validation.py`. I need to read this file to see " + "the exact implementation of the quantity validation. I'll be focusing on the code that dictates " + "whether zero or negative quantities are permitted, or how they are processed if they are not. I'll " + "also be keeping an eye out for any checks related to non-integer values, though the initial prompt " + "emphasized zero and negatives.\n\nI will then use the `read_file` tool to retrieve the content of " + "`shop/validation.py`. I'll specify a reasonable range for the lines to read, perhaps from the " + "beginning of the file up to line 50, to ensure I capture the relevant validation function and its " + "surrounding context. This will allow me to analyze the validation rules in detail before proceeding " + "to any other files.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I need to inspect `shop/validation.py` next to see how `validate_quantity` handles zero, negative, " + "and non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "id": "call_316157", + }, + "thoughtSignature": GEMINI_3_8_FLASH_RESPONSES_THOUGHT_SIGNATURE_0, + }, + ], }, "finishReason": "STOP", - } + }, ], "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 2, - "totalTokenCount": 118, + "promptTokenCount": 242, + "candidatesTokenCount": 64, + "totalTokenCount": 571, "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 2}], - "thoughtsTokenCount": 107, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 242}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 64}], + "thoughtsTokenCount": 265, }, "modelVersion": "gemini-3.8-flash", - "createTime": "2026-10-05T22:55:12.739855Z", - "responseId": "0CrEao-ULbuU9LsPk-mQ2A8", + "createTime": "2026-10-15T01:14:17.560771Z", + "responseId": "6SjQaoOdIvCfq8YP89zFuQk", }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "vertex_ai/gemini-3.8-flash", - "object": "response", "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": ( + "**My Thought Process on Inspecting `shop/validation.py`**\n\nAlright, my primary objective right now " + "is to understand how the system handles invalid quantity inputs, specifically zero, negative, and " + "potentially non-integer values. This is crucial for robust error handling and ensuring data " + "integrity within the `shop` module.\n\nTo achieve this, I need to examine the validation logic " + "directly. My instinct is that the `shop/validation.py` file will contain the relevant functions for " + "this purpose. I'm looking for something like a `validate_quantity` function, which I anticipate " + "will be located around line 18, as per my internal knowledge base or a quick scan.\n\nTherefore, the " + "next logical step is to access the content of `shop/validation.py`. I need to read this file to see " + "the exact implementation of the quantity validation. I'll be focusing on the code that dictates " + "whether zero or negative quantities are permitted, or how they are processed if they are not. I'll " + "also be keeping an eye out for any checks related to non-integer values, though the initial prompt " + "emphasized zero and negatives.\n\nI will then use the `read_file` tool to retrieve the content of " + "`shop/validation.py`. I'll specify a reasonable range for the lines to read, perhaps from the " + "beginning of the file up to line 50, to ensure I capture the relevant validation function and its " + "surrounding context. This will allow me to analyze the validation rules in detail before proceeding " + "to any other files.\n\n" + ), + "annotations": [], + }, + ], + "phase": None, + }, { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": ( + "I need to inspect `shop/validation.py` next to see how `validate_quantity` handles zero, negative, " + "and non-integer quantities." + ), + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "call_id": ( + "call_316157__thought__AY89a18zqWYsP1fCe9VRYni0L9f03IJNDHvm1boNA4Q2dVNJPzZZvrxcHGuC7kCwx5e9kBdKsGEQ9/" + "Nr4AjO5c6ukC2h2NEeTinmhak8IRJNBkDQ7OZ7at4q5KYoqYfDZ8aNohtXWBxVdoSK9YZHp+nPGYE7zmXk1icksF+WJG0ga0jBai" + "vzjYeChB7DEp/pEfxa5e/5etULNzsfurb4+l7Aht/DYaH/Z+EEttmrxrDlKI9SHGOfDy8YRn7XcDJyjYjIJ/Hgcoa3/STN5kaEJs" + "+7FzyrN64sSOyhzsP1H5dSM27anFBLUjs5KHg15CoDirSbIlRLlOHJWQ2k9Ql3ljBfNePDAds4/zH2VchSqiZfsIS5XwUGm8D7L5" + "65vzTzknZtb+zSZivDPPs/fCe3iCNYY4YEeSt2Zz/hy7baM06u6Vb1cljWtmckLoQgpiHJhY/yA5IhrHLqTx+PnBTgud6+yx7r/9" + "Wiu8ttxeU8vupqAERoBj9yOmysvjloE6N7g/9qwsRvvoxICo8luVQmA1xWqkDMTLT3LNfEwIpWmEr0Tx+y4chAZitmuKUBZdtPfr" + "ayO96GB7iPjRUarEKs/px3hHyaSs7QzZZM/ymeeihqYyWzr81GN14/MsHs72eX3sl+ooYhxM/qhpOSjFiMgffLrD7NlAwcBtXfNI" + "7fQWjbkLYf+mu6hCMSY8gshIkSLFcRv8ijH3eK2RBQWzvDsjkNgiuTlXetgYFVe5/Q3YJvCCdPsbn/Fd/dc3etq5XTGJ9B1ecsI7" + "KFdWOX8Q5HrrfWFOuo7BuWXNnFBN7pd1RwzWI3cveysx77EgQ/fMS0Iqb9FsM4i8SIY6tW5KIj+Q0SdnGR2Fkh214QxDUwT53ThP" + "Des9mapBpR4lxIqvROoMcMPoMFlcvENF9S8oeE17cifQTVFrQT6xGeWL3vV7AVGVhuP7RjIX9cU/NK2qBof50qfI2BC+aqiu6Wtz" + "WQ2IdfrhCfCChccr0Jh4IldyF/0mW45PijT0mguv0vwceuVrtwLkYSK7gsnb6REGUmQ4WXWy9VwrCmcYzmX5igAN/ml0sML0+k1p" + "DHtYVcflnjAT1s/wiQFJ4Pg1vUXmpjGSOUb7KiyKeiUvgTTdqD1EXHAWxnFJtgwwz/6p+VhkiZvvwOJY/GSciX14pMtA57VsGd+2" + "eUTRdpuEbffVWjuuyeyZd6Xgm2PNLnCIvP8sS0FLJAKhOYau1OmGwkl5gsteZqSgjg2l7hXJf3lHhXv4UrRRj/YwX8SEg/YPSzhu" + "rb3bBdLKBUPfQs3h/DlOnGbQqi7efhq5seYJBldPpiEtwChm7o6mJ6J4cr1AqLTtDmQR6XF5ZN/oE6XGHpa2XY9N4tJmSrnIq6ku" + "unngTfpD7yejAg9bHVy8r4FWjU9wU2AgYHBZneDXTiVKDaJDzYPdoufiADokGyWntWksp+B3jH+El+kI7c3uu/tL28HVep3u8Jz/" + "XlLqEgh4oViCrcsyqqjLsUrfQ8oDmiDJZfbxECR7GHDxJ9yA==" + ), + "name": "read_file", + "type": "function_call", + "id": ( + "fc_call_316157__thought__AY89a18zqWYsP1fCe9VRYni0L9f03IJNDHvm1boNA4Q2dVNJPzZZvrxcHGuC7kCwx5e9kBdKsGE" + "Q9/Nr4AjO5c6ukC2h2NEeTinmhak8IRJNBkDQ7OZ7at4q5KYoqYfDZ8aNohtXWBxVdoSK9YZHp+nPGYE7zmXk1icksF+WJG0ga0j" + "BaivzjYeChB7DEp/pEfxa5e/5etULNzsfurb4+l7Aht/DYaH/Z+EEttmrxrDlKI9SHGOfDy8YRn7XcDJyjYjIJ/Hgcoa3/STN5ka" + "EJs+7FzyrN64sSOyhzsP1H5dSM27anFBLUjs5KHg15CoDirSbIlRLlOHJWQ2k9Ql3ljBfNePDAds4/zH2VchSqiZfsIS5XwUGm8D" + "7L565vzTzknZtb+zSZivDPPs/fCe3iCNYY4YEeSt2Zz/hy7baM06u6Vb1cljWtmckLoQgpiHJhY/yA5IhrHLqTx+PnBTgud6+yx7" + "r/9Wiu8ttxeU8vupqAERoBj9yOmysvjloE6N7g/9qwsRvvoxICo8luVQmA1xWqkDMTLT3LNfEwIpWmEr0Tx+y4chAZitmuKUBZdt" + "PfrayO96GB7iPjRUarEKs/px3hHyaSs7QzZZM/ymeeihqYyWzr81GN14/MsHs72eX3sl+ooYhxM/qhpOSjFiMgffLrD7NlAwcBtX" + "fNI7fQWjbkLYf+mu6hCMSY8gshIkSLFcRv8ijH3eK2RBQWzvDsjkNgiuTlXetgYFVe5/Q3YJvCCdPsbn/Fd/dc3etq5XTGJ9B1ec" + "sI7KFdWOX8Q5HrrfWFOuo7BuWXNnFBN7pd1RwzWI3cveysx77EgQ/fMS0Iqb9FsM4i8SIY6tW5KIj+Q0SdnGR2Fkh214QxDUwT53" + "ThPDes9mapBpR4lxIqvROoMcMPoMFlcvENF9S8oeE17cifQTVFrQT6xGeWL3vV7AVGVhuP7RjIX9cU/NK2qBof50qfI2BC+aqiu6" + "WtzWQ2IdfrhCfCChccr0Jh4IldyF/0mW45PijT0mguv0vwceuVrtwLkYSK7gsnb6REGUmQ4WXWy9VwrCmcYzmX5igAN/ml0sML0+" + "k1pDHtYVcflnjAT1s/wiQFJ4Pg1vUXmpjGSOUb7KiyKeiUvgTTdqD1EXHAWxnFJtgwwz/6p+VhkiZvvwOJY/GSciX14pMtA57VsG" + "d+2eUTRdpuEbffVWjuuyeyZd6Xgm2PNLnCIvP8sS0FLJAKhOYau1OmGwkl5gsteZqSgjg2l7hXJf3lHhXv4UrRRj/YwX8SEg/YPS" + "zhurb3bBdLKBUPfQs3h/DlOnGbQqi7efhq5seYJBldPpiEtwChm7o6mJ6J4cr1AqLTtDmQR6XF5ZN/oE6XGHpa2XY9N4tJmSrnIq" + "6kuunngTfpD7yejAg9bHVy8r4FWjU9wU2AgYHBZneDXTiVKDaJDzYPdoufiADokGyWntWksp+B3jH+El+kI7c3uu/tL28HVep3u8" + "Jz/XlLqEgh4oViCrcsyqqjLsUrfQ8oDmiDJZfbxECR7GHDxJ9yA==" + ), + "namespace": None, + "status": "completed", + "provider_specific_fields": { + "thought_signature": ( + "AY89a18zqWYsP1fCe9VRYni0L9f03IJNDHvm1boNA4Q2dVNJPzZZvrxcHGuC7kCwx5e9kBdKsGEQ9/Nr4AjO5c6ukC2h2NEeTinm" + "hak8IRJNBkDQ7OZ7at4q5KYoqYfDZ8aNohtXWBxVdoSK9YZHp+nPGYE7zmXk1icksF+WJG0ga0jBaivzjYeChB7DEp/pEfxa5e/5" + "etULNzsfurb4+l7Aht/DYaH/Z+EEttmrxrDlKI9SHGOfDy8YRn7XcDJyjYjIJ/Hgcoa3/STN5kaEJs+7FzyrN64sSOyhzsP1H5dS" + "M27anFBLUjs5KHg15CoDirSbIlRLlOHJWQ2k9Ql3ljBfNePDAds4/zH2VchSqiZfsIS5XwUGm8D7L565vzTzknZtb+zSZivDPPs/" + "fCe3iCNYY4YEeSt2Zz/hy7baM06u6Vb1cljWtmckLoQgpiHJhY/yA5IhrHLqTx+PnBTgud6+yx7r/9Wiu8ttxeU8vupqAERoBj9y" + "OmysvjloE6N7g/9qwsRvvoxICo8luVQmA1xWqkDMTLT3LNfEwIpWmEr0Tx+y4chAZitmuKUBZdtPfrayO96GB7iPjRUarEKs/px3" + "hHyaSs7QzZZM/ymeeihqYyWzr81GN14/MsHs72eX3sl+ooYhxM/qhpOSjFiMgffLrD7NlAwcBtXfNI7fQWjbkLYf+mu6hCMSY8gs" + "hIkSLFcRv8ijH3eK2RBQWzvDsjkNgiuTlXetgYFVe5/Q3YJvCCdPsbn/Fd/dc3etq5XTGJ9B1ecsI7KFdWOX8Q5HrrfWFOuo7BuW" + "XNnFBN7pd1RwzWI3cveysx77EgQ/fMS0Iqb9FsM4i8SIY6tW5KIj+Q0SdnGR2Fkh214QxDUwT53ThPDes9mapBpR4lxIqvROoMcM" + "PoMFlcvENF9S8oeE17cifQTVFrQT6xGeWL3vV7AVGVhuP7RjIX9cU/NK2qBof50qfI2BC+aqiu6WtzWQ2IdfrhCfCChccr0Jh4Il" + "dyF/0mW45PijT0mguv0vwceuVrtwLkYSK7gsnb6REGUmQ4WXWy9VwrCmcYzmX5igAN/ml0sML0+k1pDHtYVcflnjAT1s/wiQFJ4P" + "g1vUXmpjGSOUb7KiyKeiUvgTTdqD1EXHAWxnFJtgwwz/6p+VhkiZvvwOJY/GSciX14pMtA57VsGd+2eUTRdpuEbffVWjuuyeyZd6" + "Xgm2PNLnCIvP8sS0FLJAKhOYau1OmGwkl5gsteZqSgjg2l7hXJf3lHhXv4UrRRj/YwX8SEg/YPSzhurb3bBdLKBUPfQs3h/DlOnG" + "bQqi7efhq5seYJBldPpiEtwChm7o6mJ6J4cr1AqLTtDmQR6XF5ZN/oE6XGHpa2XY9N4tJmSrnIq6kuunngTfpD7yejAg9bHVy8r4" + "FWjU9wU2AgYHBZneDXTiVKDaJDzYPdoufiADokGyWntWksp+B3jH+El+kI7c3uu/tL28HVep3u8Jz/XlLqEgh4oViCrcsyqqjLsU" + "rfQ8oDmiDJZfbxECR7GHDxJ9yA==" + ), + }, + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 9, + "input_tokens": 242, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 9, + "text_tokens": 242, "video_tokens": None, }, - "output_tokens": 109, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 107, "text_tokens": 2}, - "total_tokens": 118, + "output_tokens": 329, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 265, "text_tokens": 64}, + "total_tokens": 571, "cost": None, }, - "user": None, - "store": None, "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, }, ) + GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "vertex_ai/gemini-3.1-pro-preview", - "max_output_tokens": 1024, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "vertex_ai/gemini-3.1-pro-preview"}, expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-3.1-pro-preview:generateContent", expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, expected_provider_request={ - "contents": [{"role": "user", "parts": [{"text": "Say hello."}]}], - "system_instruction": {"parts": [{"text": "You are a terse assistant."}]}, - "generationConfig": {"max_output_tokens": 1024}, + **VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the responses " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + }, }, mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, "candidates": [ { "content": { "role": "model", - "parts": [{"text": "Hello.", "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_THOUGHT_SIGNATURE}], + "parts": [ + { + "text": ( + "**My Thought Process on Validating Quantities**\n\nAlright, so my primary objective here is to get a " + "solid grasp on how the system validates quantities. I need to specifically understand its behavior " + "when faced with zero or negative values, as I've observed that the checkout process currently " + "rejects zero quantities, and I want to confirm if that's a deliberate design choice or something to " + "be revisited.\n\nMy first step is to pinpoint the exact location of this validation logic. Based on " + "the information I have, it seems the relevant function, `validate_quantity`, resides within the " + "`shop/validation.py` file, specifically around line 18.\n\nTherefore, my immediate action is to dive " + "into `shop/validation.py`. I'll focus on the vicinity of line 18, reading a few lines before and " + "after – let's say lines 10 through 30 – to ensure I have sufficient context to understand the " + "function's implementation and any surrounding logic.\n\nOnce I've examined the code, I'll be able to " + "articulate precisely how `validate_quantity` handles zero, negative, and potentially even " + "non-integer quantities. Then, I can proceed with the next steps, whether that involves reporting my " + "findings, proposing modifications, or simply confirming the current state of affairs. So, my next " + "move is to execute that `read_file` operation.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I need to look at `shop/validation.py` to see the `validate_quantity` function and understand its " + "logic for zero, negative, and non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"path": "shop/validation.py", "start_line": 10, "end_line": 30}, + "id": "call_226688", + }, + "thoughtSignature": GEMINI_3_1_PRO_PREVIEW_RESPONSES_THOUGHT_SIGNATURE_0, + }, + ], }, "finishReason": "STOP", - } + }, ], "usageMetadata": { - "promptTokenCount": 9, - "candidatesTokenCount": 1, - "totalTokenCount": 99, + "promptTokenCount": 242, + "candidatesTokenCount": 70, + "totalTokenCount": 483, "trafficType": "ON_DEMAND", - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 9}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 1}], - "thoughtsTokenCount": 89, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 242}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 70}], + "thoughtsTokenCount": 171, }, "modelVersion": "gemini-3.1-pro-preview", - "createTime": "2026-10-05T22:55:14.874538Z", - "responseId": "0irEaqqwNbCdq8YP3L2mqQ4", + "createTime": "2026-10-15T01:14:11.647331Z", + "responseId": "4yjQaqPBJ72apucPwee_YQ", }, - expected_litellm_status_code=200, expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, + **RESPONSES_RESPONSE, "model": "vertex_ai/gemini-3.1-pro-preview", - "object": "response", "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": ( + "**My Thought Process on Validating Quantities**\n\nAlright, so my primary objective here is to get a " + "solid grasp on how the system validates quantities. I need to specifically understand its behavior " + "when faced with zero or negative values, as I've observed that the checkout process currently " + "rejects zero quantities, and I want to confirm if that's a deliberate design choice or something to " + "be revisited.\n\nMy first step is to pinpoint the exact location of this validation logic. Based on " + "the information I have, it seems the relevant function, `validate_quantity`, resides within the " + "`shop/validation.py` file, specifically around line 18.\n\nTherefore, my immediate action is to dive " + "into `shop/validation.py`. I'll focus on the vicinity of line 18, reading a few lines before and " + "after – let's say lines 10 through 30 – to ensure I have sufficient context to understand the " + "function's implementation and any surrounding logic.\n\nOnce I've examined the code, I'll be able to " + "articulate precisely how `validate_quantity` handles zero, negative, and potentially even " + "non-integer quantities. Then, I can proceed with the next steps, whether that involves reporting my " + "findings, proposing modifications, or simply confirming the current state of affairs. So, my next " + "move is to execute that `read_file` operation.\n\n" + ), + "annotations": [], + }, + ], + "phase": None, + }, { "type": "message", "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], + "content": [ + { + "type": "output_text", + "text": ( + "I need to look at `shop/validation.py` to see the `validate_quantity` function and understand its " + "logic for zero, negative, and non-integer quantities." + ), + "annotations": [], + }, + ], "phase": None, - } + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 10, "end_line": 30}', + "call_id": ( + "call_226688__thought__AY89a18mH4gPpBuF8sZq3BxEt7fVfeky9tDk6WrFiAYqUNfy1TwKL5UEuc60FQFzZzoY6BhtmNYM/Y" + "nvWntgJG06tpD+R69L+px8wJWN2jf96nYOxDNe5xM3Z6yz7QO1XikWun/bqStP3GVvN2xCTRBeQxkN0n5+NPDWYXC94zo2uikRYf" + "1XZ5IsCrbwhDgFhqzyv+3CpEmBpN7ghzonZp6GL23N0ELNi95QEzZMcGrD31Z3xx6Db+KblQL9yVKVj7STHJ4Bno3sU6heJx9NMJ" + "bM55wI5SNcZqi8YqkS3KliWRxcPdWbkekAOCWUFYYFG0JEB0iOibkqcCV6j7bZaJD3v9qdqpaSbG1Q3yl5Ncb1KMMGLwbMyjN9t3" + "XrFQuqtM1x3OE8hofM3bNPMYGna3DDEmlporSsQlE6W8Cxa70IsBaFY5h7eQjKCkxLSOhZmGfmuukLLpxXWG8ZHzdtOB7vmpIcBo" + "lW2TcQCamojGpvWdxKz/JhU9WB9xac6A5luWXnIA+YC/Pu793XzkDrST8jT4bei8Wi37sfAhRUkVeUw6VAAYbZKONNM43U/ih3Bw" + "JvpJoY5voZVyhm4JTT66q9vSTUSiR4oQz/ipXDUKVfWFhqEWoh3Nh5elhFe1oZyx8q+lGwXm4VcrGNtb5xwWGnAsQN9GV30egFGN" + "cEKFq3rnODehteSTTaHF6YaqklMypgS/K09SfshF0waKvX0GDouVMbnO3bFfMxGU1CofWB3b3q6RbIdv+O37WekKo0O8S6OwQRdp" + "fOCNXcwpwsRczX4s9IoGRK5IIKr1/6cQze7LA2Gi00gBIe9RpbSMJHo/XJyDsP6KT1IAAtoEkyGBzsVjBEU4ggVsG6n8ka5y4myF" + "gwMQeNrN5DNqFL+nVzxqUF97pRU7Xw37SlNNi9DgGnqhyWY/T6Vvr4yhCsYR2BElGCq0C1YUF9OQ8+gp2CBrtd+bRLHz5iNyUMoP" + "umSAn6B1+GyU5JFPQwC9YvNK6ug+9s2akQgYYgNAnDS7aXj2djQ57nwPqnsg==" + ), + "name": "read_file", + "type": "function_call", + "id": ( + "fc_call_226688__thought__AY89a18mH4gPpBuF8sZq3BxEt7fVfeky9tDk6WrFiAYqUNfy1TwKL5UEuc60FQFzZzoY6BhtmNY" + "M/YnvWntgJG06tpD+R69L+px8wJWN2jf96nYOxDNe5xM3Z6yz7QO1XikWun/bqStP3GVvN2xCTRBeQxkN0n5+NPDWYXC94zo2uik" + "RYf1XZ5IsCrbwhDgFhqzyv+3CpEmBpN7ghzonZp6GL23N0ELNi95QEzZMcGrD31Z3xx6Db+KblQL9yVKVj7STHJ4Bno3sU6heJx9" + "NMJbM55wI5SNcZqi8YqkS3KliWRxcPdWbkekAOCWUFYYFG0JEB0iOibkqcCV6j7bZaJD3v9qdqpaSbG1Q3yl5Ncb1KMMGLwbMyjN" + "9t3XrFQuqtM1x3OE8hofM3bNPMYGna3DDEmlporSsQlE6W8Cxa70IsBaFY5h7eQjKCkxLSOhZmGfmuukLLpxXWG8ZHzdtOB7vmpI" + "cBolW2TcQCamojGpvWdxKz/JhU9WB9xac6A5luWXnIA+YC/Pu793XzkDrST8jT4bei8Wi37sfAhRUkVeUw6VAAYbZKONNM43U/ih" + "3BwJvpJoY5voZVyhm4JTT66q9vSTUSiR4oQz/ipXDUKVfWFhqEWoh3Nh5elhFe1oZyx8q+lGwXm4VcrGNtb5xwWGnAsQN9GV30eg" + "FGNcEKFq3rnODehteSTTaHF6YaqklMypgS/K09SfshF0waKvX0GDouVMbnO3bFfMxGU1CofWB3b3q6RbIdv+O37WekKo0O8S6OwQ" + "RdpfOCNXcwpwsRczX4s9IoGRK5IIKr1/6cQze7LA2Gi00gBIe9RpbSMJHo/XJyDsP6KT1IAAtoEkyGBzsVjBEU4ggVsG6n8ka5y4" + "myFgwMQeNrN5DNqFL+nVzxqUF97pRU7Xw37SlNNi9DgGnqhyWY/T6Vvr4yhCsYR2BElGCq0C1YUF9OQ8+gp2CBrtd+bRLHz5iNyU" + "MoPumSAn6B1+GyU5JFPQwC9YvNK6ug+9s2akQgYYgNAnDS7aXj2djQ57nwPqnsg==" + ), + "namespace": None, + "status": "completed", + "provider_specific_fields": { + "thought_signature": ( + "AY89a18mH4gPpBuF8sZq3BxEt7fVfeky9tDk6WrFiAYqUNfy1TwKL5UEuc60FQFzZzoY6BhtmNYM/YnvWntgJG06tpD+R69L+px8" + "wJWN2jf96nYOxDNe5xM3Z6yz7QO1XikWun/bqStP3GVvN2xCTRBeQxkN0n5+NPDWYXC94zo2uikRYf1XZ5IsCrbwhDgFhqzyv+3C" + "pEmBpN7ghzonZp6GL23N0ELNi95QEzZMcGrD31Z3xx6Db+KblQL9yVKVj7STHJ4Bno3sU6heJx9NMJbM55wI5SNcZqi8YqkS3Kli" + "WRxcPdWbkekAOCWUFYYFG0JEB0iOibkqcCV6j7bZaJD3v9qdqpaSbG1Q3yl5Ncb1KMMGLwbMyjN9t3XrFQuqtM1x3OE8hofM3bNP" + "MYGna3DDEmlporSsQlE6W8Cxa70IsBaFY5h7eQjKCkxLSOhZmGfmuukLLpxXWG8ZHzdtOB7vmpIcBolW2TcQCamojGpvWdxKz/Jh" + "U9WB9xac6A5luWXnIA+YC/Pu793XzkDrST8jT4bei8Wi37sfAhRUkVeUw6VAAYbZKONNM43U/ih3BwJvpJoY5voZVyhm4JTT66q9" + "vSTUSiR4oQz/ipXDUKVfWFhqEWoh3Nh5elhFe1oZyx8q+lGwXm4VcrGNtb5xwWGnAsQN9GV30egFGNcEKFq3rnODehteSTTaHF6Y" + "aqklMypgS/K09SfshF0waKvX0GDouVMbnO3bFfMxGU1CofWB3b3q6RbIdv+O37WekKo0O8S6OwQRdpfOCNXcwpwsRczX4s9IoGRK" + "5IIKr1/6cQze7LA2Gi00gBIe9RpbSMJHo/XJyDsP6KT1IAAtoEkyGBzsVjBEU4ggVsG6n8ka5y4myFgwMQeNrN5DNqFL+nVzxqUF" + "97pRU7Xw37SlNNi9DgGnqhyWY/T6Vvr4yhCsYR2BElGCq0C1YUF9OQ8+gp2CBrtd+bRLHz5iNyUMoPumSAn6B1+GyU5JFPQwC9Yv" + "NK6ug+9s2akQgYYgNAnDS7aXj2djQ57nwPqnsg==" + ), + }, + }, ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 1024, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, "usage": { - "input_tokens": 9, + "input_tokens": 242, "input_tokens_details": { "audio_tokens": None, "cached_tokens": 0, "cached_tokens_details": None, "image_tokens": None, - "text_tokens": 9, + "text_tokens": 242, "video_tokens": None, }, - "output_tokens": 90, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 89, "text_tokens": 1}, - "total_tokens": 99, + "output_tokens": 241, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 171, "text_tokens": 70}, + "total_tokens": 483, + "cost": None, + }, + "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, + }, +) + + +GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "vertex_ai/gemini-4-argon"}, + expected_provider_endpoint="/v1/projects/synthetic-vertex-project/locations/global/publishers/google/models/gemini-4-argon:generateContent", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer scripted-token"}, + expected_provider_request={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST, + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the responses " + "client for this project review. Consider zero, negative and non-integer quantities before choosing " + "the next file to inspect." + ), + }, + ], + }, + }, + mock_provider_response={ + **VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE, + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": ( + "**My Thought Process for Analyzing `shop/validation.py`**\n\nMy immediate priority is to gain a " + "thorough understanding of how the system handles invalid product quantities. Specifically, I need " + "to know how it's designed to process zero, negative, and non-integer values. To achieve this, my " + "next step is to examine the `shop/validation.py` file. The developer's prompt explicitly states " + 'that I should "Explain which file you need next, then call a read-only tool. Never change files." ' + "Therefore, I will proceed by identifying `shop/validation.py` as the crucial file for this analysis " + "and then use the `read_file` tool to access its contents.\n\n" + ), + "thought": True, + }, + { + "text": ( + "I will read `shop/validation.py` next to inspect how `validate_quantity` handles zero, negative, " + "and non-integer quantities." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"path": "shop/validation.py", "end_line": 50, "start_line": 1}, + "id": "call_268046", + }, + "thoughtSignature": GEMINI_4_ARGON_RESPONSES_THOUGHT_SIGNATURE_0, + }, + ], + }, + "finishReason": "STOP", + }, + ], + "usageMetadata": { + "promptTokenCount": 412, + "candidatesTokenCount": 69, + "totalTokenCount": 573, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 69}], + "thoughtsTokenCount": 92, + }, + "modelVersion": "gemini-4-argon", + "createTime": "2026-10-15T01:14:21.864566Z", + "responseId": "7SjQarbiNNqX9LsPrO6x6QE", + }, + expected_litellm_response={ + **RESPONSES_RESPONSE, + "model": "vertex_ai/gemini-4-argon", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": ( + "**My Thought Process for Analyzing `shop/validation.py`**\n\nMy immediate priority is to gain a " + "thorough understanding of how the system handles invalid product quantities. Specifically, I need " + "to know how it's designed to process zero, negative, and non-integer values. To achieve this, my " + "next step is to examine the `shop/validation.py` file. The developer's prompt explicitly states " + 'that I should "Explain which file you need next, then call a read-only tool. Never change files." ' + "Therefore, I will proceed by identifying `shop/validation.py` as the crucial file for this analysis " + "and then use the `read_file` tool to access its contents.\n\n" + ), + "annotations": [], + }, + ], + "phase": None, + }, + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": ( + "I will read `shop/validation.py` next to inspect how `validate_quantity` handles zero, negative, " + "and non-integer quantities." + ), + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "end_line": 50, "start_line": 1}', + "call_id": ( + "call_268046__thought__AY89a180ZmYN6Zq/ICj0BCu+eBS75TWGlATeX2fsIhoKaFuGLfTjmizARO/YOhSzt4TK8d1jl7skKb" + "Dq23TdvRVOJfjPsI85UiVCF8DZC4gSdHQbTXiDtBUsue7BxMBnoNfPJe34Y1p6WqjbaK44zsfJybZVsWJSmQOW5Ky8qnBZXfEL9d" + "yz2j0rQbN5IMGUV36wrA4vKLfUj6vS1dwzz2LETuK1rT/EFKXDG2Pwgo11yMzoZuPvC8VvNrQXqJ/Tizd/cuuc+Fiht7V4EvrEDn" + "DH0ZFMW4gyS52mfjE8/hnIFIgzaFkhP7mWHBOwEt3ZSPZPQ2d6ZEOXD+1clQXSOWMv9Ks4Iwvy8hrcri60cdqbFdbHxF48OgfWUy" + "2dUycP6kuKwWUJ+sDNT1pNlb6pfhKdmP7wd6QvjudKrrunGZbWr9CjkiAGVOD8RfcQK8Z7LUC11RyHqGDXLxJA1kzR4W4wct1Y6u" + "WF3TRN8QsSLpuXLS+sITYsvEaRtT/wcCBVYu+dHephEAo2UXmaMfkZ8QxrWtf/VOwDZegQiVpBkxOZyAoByiDJrCwFa/osvXXcj0" + "BtweIKpKKUdLy77BVE7IHaJikXSIRmpwYZRnA=" + ), + "name": "read_file", + "type": "function_call", + "id": ( + "fc_call_268046__thought__AY89a180ZmYN6Zq/ICj0BCu+eBS75TWGlATeX2fsIhoKaFuGLfTjmizARO/YOhSzt4TK8d1jl7s" + "kKbDq23TdvRVOJfjPsI85UiVCF8DZC4gSdHQbTXiDtBUsue7BxMBnoNfPJe34Y1p6WqjbaK44zsfJybZVsWJSmQOW5Ky8qnBZXfE" + "L9dyz2j0rQbN5IMGUV36wrA4vKLfUj6vS1dwzz2LETuK1rT/EFKXDG2Pwgo11yMzoZuPvC8VvNrQXqJ/Tizd/cuuc+Fiht7V4Evr" + "EDnDH0ZFMW4gyS52mfjE8/hnIFIgzaFkhP7mWHBOwEt3ZSPZPQ2d6ZEOXD+1clQXSOWMv9Ks4Iwvy8hrcri60cdqbFdbHxF48Ogf" + "WUy2dUycP6kuKwWUJ+sDNT1pNlb6pfhKdmP7wd6QvjudKrrunGZbWr9CjkiAGVOD8RfcQK8Z7LUC11RyHqGDXLxJA1kzR4W4wct1" + "Y6uWF3TRN8QsSLpuXLS+sITYsvEaRtT/wcCBVYu+dHephEAo2UXmaMfkZ8QxrWtf/VOwDZegQiVpBkxOZyAoByiDJrCwFa/osvXX" + "cj0BtweIKpKKUdLy77BVE7IHaJikXSIRmpwYZRnA=" + ), + "namespace": None, + "status": "completed", + "provider_specific_fields": { + "thought_signature": ( + "AY89a180ZmYN6Zq/ICj0BCu+eBS75TWGlATeX2fsIhoKaFuGLfTjmizARO/YOhSzt4TK8d1jl7skKbDq23TdvRVOJfjPsI85UiVC" + "F8DZC4gSdHQbTXiDtBUsue7BxMBnoNfPJe34Y1p6WqjbaK44zsfJybZVsWJSmQOW5Ky8qnBZXfEL9dyz2j0rQbN5IMGUV36wrA4v" + "KLfUj6vS1dwzz2LETuK1rT/EFKXDG2Pwgo11yMzoZuPvC8VvNrQXqJ/Tizd/cuuc+Fiht7V4EvrEDnDH0ZFMW4gyS52mfjE8/hnI" + "FIgzaFkhP7mWHBOwEt3ZSPZPQ2d6ZEOXD+1clQXSOWMv9Ks4Iwvy8hrcri60cdqbFdbHxF48OgfWUy2dUycP6kuKwWUJ+sDNT1pN" + "lb6pfhKdmP7wd6QvjudKrrunGZbWr9CjkiAGVOD8RfcQK8Z7LUC11RyHqGDXLxJA1kzR4W4wct1Y6uWF3TRN8QsSLpuXLS+sITYs" + "vEaRtT/wcCBVYu+dHephEAo2UXmaMfkZ8QxrWtf/VOwDZegQiVpBkxOZyAoByiDJrCwFa/osvXXcj0BtweIKpKKUdLy77BVE7IHa" + "JikXSIRmpwYZRnA=" + ), + }, + }, + ], + "usage": { + "input_tokens": 412, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 412, + "video_tokens": None, + }, + "output_tokens": 161, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 92, "text_tokens": 69}, + "total_tokens": 573, "cost": None, }, - "user": None, - "store": None, "provider_specific_fields": {"traffic_type": "ON_DEMAND"}, }, ) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_anthropic.py b/tests/integration/translation/responses/basic/test_responses_basic_anthropic.py index 776454b0734..ab9a35d0e66 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_anthropic.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_anthropic.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/responses/basic/test_responses_basic_azure.py b/tests/integration/translation/responses/basic/test_responses_basic_azure.py index b7f42da50f5..e2547778c52 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_azure.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_azure.py @@ -6,6 +6,7 @@ from integration.translation.responses.bases.azure import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -22,6 +23,7 @@ from integration.translation.runner import assert_translation GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_azure_ai.py b/tests/integration/translation/responses/basic/test_responses_basic_azure_ai.py index 503a44c0827..aa83983c389 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_azure_ai.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_azure_ai.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py index bbce3d53cb8..b45b15508ed 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_converse.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/responses/basic/test_responses_basic_bedrock_invoke.py b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_invoke.py index d07ccc6b515..7f77929f20b 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_bedrock_invoke.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_invoke.py @@ -19,12 +19,12 @@ from integration.translation.runner import assert_translation "case", [ CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_FABLE_5_1_TEST_CASE, ], ids=lambda case: case.id, diff --git a/tests/integration/translation/responses/basic/test_responses_basic_bedrock_mantle.py b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_mantle.py index fc4a7289adb..01d1ec9c433 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_bedrock_mantle.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_bedrock_mantle.py @@ -9,7 +9,9 @@ from integration.translation.responses.bases.bedrock_mantle import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, + GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, ) from integration.translation.runner import assert_translation @@ -26,6 +28,8 @@ from integration.translation.runner import assert_translation GPT_5_6_LUNA_TEST_CASE, GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, + GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_gemini.py b/tests/integration/translation/responses/basic/test_responses_basic_gemini.py index c6177f97336..b78268e08b0 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_gemini.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_gemini.py @@ -6,13 +6,19 @@ from integration.translation.responses.bases.gemini import ( GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ) from integration.translation.runner import assert_translation @pytest.mark.parametrize( "case", - [GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE], + [ + GEMINI_3_5_FLASH_TEST_CASE, + GEMINI_3_8_FLASH_TEST_CASE, + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, + ], ids=lambda case: case.id, ) def test_responses_basic_gemini(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: diff --git a/tests/integration/translation/responses/basic/test_responses_basic_openai.py b/tests/integration/translation/responses/basic/test_responses_basic_openai.py index 8a303e4c06f..0d5e46237d2 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_openai.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_openai.py @@ -6,6 +6,7 @@ from integration.translation.responses.bases.openai import ( GPT_5_4_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -22,6 +23,7 @@ from integration.translation.runner import assert_translation GPT_6_LUNA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, + GPT_5_6_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py b/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py index cfae3e3f7f6..67c3cf4f7a9 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_vertex_ai.py @@ -1,19 +1,19 @@ import pytest - from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase from integration.translation.responses.bases.vertex_ai import ( CLAUDE_FABLE_5_1_TEST_CASE, CLAUDE_HAIKU_4_5_TEST_CASE, - CLAUDE_SONNET_4_6_TEST_CASE, - CLAUDE_SONNET_5_TEST_CASE, CLAUDE_OPUS_4_8_TEST_CASE, - CLAUDE_SONNET_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, + CLAUDE_SONNET_4_6_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, + CLAUDE_SONNET_5_TEST_CASE, + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, - GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ) from integration.translation.runner import assert_translation @@ -31,6 +31,7 @@ from integration.translation.runner import assert_translation GEMINI_3_5_FLASH_TEST_CASE, GEMINI_3_8_FLASH_TEST_CASE, GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_4_ARGON_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_azure_ai.py b/tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_azure_ai.py new file mode 100644 index 00000000000..336e4d53ab8 --- /dev/null +++ b/tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_azure_ai.py @@ -0,0 +1,210 @@ +import json +from dataclasses import replace +from typing import Final +from unittest.mock import ANY + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.responses.bases.azure_ai import CLAUDE_FABLE_5_1_TEST_CASE, CLAUDE_HAIKU_5_5_TEST_CASE +from integration.translation.runner import assert_translation + +CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0: Final = ( + "CAQSigYKEAgSGAI4AUIIdGhpbmtpbmcSDN/XoZMwEppAeWDGmxoM/11DgkoGMFpDINNAIjCfFDmj3tewcz8c9UjL+01psTk6E1rlkSIYmcJH" + "Hx6XtPWsgJDUgKSz2IfPM/CADUMqpwUIur1FUoHdWfUasx1dqp3FuKO96MPj2UY+TH5Youusp0VbxMwTm1f03+bmK6mZkZkmgwk5qRgj56vT" + "p6vd99GSvO3AOVdjBQrAU8m1VT1Npw1cDxDa2a0TZPxJDaAt6j2M0ujvcNHE+d3KEFsC4HTJA6qIlYJrgbuOYH90PhqVCh7TjlEpNQ/OMZ/l" + "vSVBVvxVugSXAn01U1fhPE40RYuQxFzoAa5v0wvHzEjVOFV+piv/Wbm4QkNVxQ3OmEGuFwLAwK18XHQjqkqchQDctMJIiODTCu2s8mtTXGfX" + "fC2Zowpyt+k8csmXbZa0QVaVY8EpmmaDv2aeunUgiPehivaXJHBl1fJA9UB9XxmQhfappJHNHvHS78GcFfRRflv4TrcpVBUFYzoerw3PKN2U" + "vkaAwV0MlnLeOoAPsG0UcJm+ETTfh/ZbR74vPBDiFy3BSmsqQHjbBk4OGUsV15/9+/2ftIkv/2+j+4kP2wZDlMTHAwY7nNwBhdgjpdJ1Ty8K" + "/PnzHGjNocCjLIaupO85mXFPXBLY5vsS2tq4OirPRHFrrsYOKIMY49oBVGAYcRxgFh++Zc2P7ENgrGB1AuPJmTmIN2srt2kjXfdldffHRina" + "GeImXB1+G31+7CHmZvzeicDCUa8V3rl1h2eXufnHT9ZoGBERbzfPzW/f99EWT4UxTrPO6beZR5U7KxS6gb+B48j+6NvOuVsIOQfCt4wUxaHQ" + "W3w8mBXZrmjxcZzRq7sgDu47pvb33GddMF22wY6ZKy9i1bfCqxewdM9CdAJATJ6U+YJDwg1bqon5D+LjernnjPgk8pXpIWewLdt0kHVAFQQz" + "o4xTC0ofJp3L8/xNaxwmLFJ6P43o7VHsxWRKmGq/6XSJQJv9psAnccAhIF4dWxMaMHioSfj1GAE=" +) + +CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_HAIKU_5_5_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_HAIKU_5_5_TEST_CASE.mock_provider_response, + "id": "msg_011Cg37cq1Hecba5uoPsLUid", + "content": [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py to check its contents, starting with the first 60 lines " + "or so since I don't know the file's full length yet.\n\n", + "signature": CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "tool_use", + "id": "toolu_01MnZE6CSPdC7bHqEiAaKehz", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 873, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 175, + "output_tokens_details": {"thinking_tokens": 75}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **CLAUDE_HAIKU_5_5_TEST_CASE.expected_litellm_response, + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I'll read shop/validation.py to check its contents, starting with the first 60 lines " + "or so since I don't know the file's full length yet.\n\n", + "annotations": [], + }, + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py to check its contents, starting with the first 60 lines " + "or so since I don't know the file's full length yet.\n\n", + "signature": CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_01MnZE6CSPdC7bHqEiAaKehz", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01MnZE6CSPdC7bHqEiAaKehz", + "namespace": None, + "status": "completed", + }, + ], + "usage": { + "input_tokens": 877, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 873, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 175, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 75, "text_tokens": 100}, + "total_tokens": 1052, + "cost": None, + }, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll read shop/validation.py to check its contents, starting with the first 60 lines " + "or so since I don't know the file's full length yet.\n\n", + "signature": CLAUDE_HAIKU_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + ], + }, + }, +) + +CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_FABLE_5_1_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_FABLE_5_1_TEST_CASE.mock_provider_response, + "id": "msg_011Cg37VYYdzvnhETMSHzcVV", + "content": [ + {"type": "text", "text": "I'll read the validation function in `shop/validation.py`."}, + { + "type": "tool_use", + "id": "toolu_01SamuxVRxpP9kLotKTnYMfn", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 873, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 119, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "not_available", + }, + }, + expected_litellm_response={ + **CLAUDE_FABLE_5_1_TEST_CASE.expected_litellm_response, + "output": [ + { + "type": "message", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "I'll read the validation function in `shop/validation.py`.", + "annotations": [], + }, + ], + "phase": None, + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_01SamuxVRxpP9kLotKTnYMfn", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01SamuxVRxpP9kLotKTnYMfn", + "namespace": None, + "status": "completed", + }, + ], + "usage": { + "input_tokens": 877, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 873, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 119, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 119}, + "total_tokens": 996, + "cost": None, + }, + "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + }, +) + + +@pytest.mark.parametrize( + "case", + [CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE, CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE], + ids=[CLAUDE_HAIKU_5_5_CACHE_HIT_TEST_CASE.id, CLAUDE_FABLE_5_1_CACHE_HIT_TEST_CASE.id], +) +def test_responses_prompt_cache_hit_azure_ai( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/responses/templates.py b/tests/integration/translation/responses/templates.py index 70ae996e785..bb3eee37e5d 100644 --- a/tests/integration/translation/responses/templates.py +++ b/tests/integration/translation/responses/templates.py @@ -285,3 +285,8 @@ RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { ], }, } + + +RESPONSES_NATIVE_RESPONSE: Final[Mapping[str, JsonValue]] = { + key: value for key, value in RESPONSES_RESPONSE.items() if key not in {"provider_specific_fields"} +} diff --git a/tests/integration/translation/templates/azure.py b/tests/integration/translation/templates/azure.py new file mode 100644 index 00000000000..dfe52a1e4db --- /dev/null +++ b/tests/integration/translation/templates/azure.py @@ -0,0 +1,354 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +AZURE_RESPONSES_REQUEST: Final[Mapping[str, JsonValue]] = { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": ("The checkout rejects quantities of zero. Find the validation code and its tests."), + } + ], + }, + { + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": ( + '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", "line": 7, ' + '"text": "def test_zero_quantity():"}]}' + ), + }, + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": ( + "Read the validation file first so we can understand how zero and negative " + "quantities are handled." + ), + } + ], + }, + ], + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect." + ), + "max_output_tokens": 1024, + "reasoning": {"effort": "high"}, + "tools": [ + { + "type": "function", + "name": "search_files", + "description": "Search source and test files for a literal term.", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + { + "type": "function", + "name": "read_file", + "description": "Read a range of lines from a project file.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + ], +} + +AZURE_RESPONSES_BRIDGE_REQUEST: Final[Mapping[str, JsonValue]] = { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": ("The checkout rejects quantities of zero. Find the validation code and its tests."), + } + ], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": [ + { + "type": "input_text", + "text": ( + '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", "line": 7, ' + '"text": "def test_zero_quantity():"}]}' + ), + } + ], + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": ( + "Read the validation file first so we can understand how zero and negative " + "quantities are handled." + ), + } + ], + }, + ], + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the chat_completions client for this project review. " + "Consider zero, negative and non-integer quantities before choosing the next file " + "to inspect." + ), + "max_output_tokens": 1024, + "reasoning": {"effort": "high"}, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "description": "Search source and test files for a literal term.", + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "description": "Read a range of lines from a project file.", + }, + ], +} + +AZURE_RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { + "id": "resp_03952e888f4f1827006ad023aa00108190b24e51d376e9a373", + "object": "response", + "created_at": 1792025514, + "status": "completed", + "background": False, + "completed_at": 1792025519, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 575, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 578, "check_offset": 0}, + }, + ], + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": ( + "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the chat_completions client for this project review. " + "Consider zero, negative and non-integer quantities before choosing the next file " + "to inspect." + ), + "max_output_tokens": 1024, + "max_tool_calls": None, + "model": "gpt-6.1-sol", + "moderation": None, + "output": [ + { + "id": "rs_03952e888f4f1827006ad023ae0c048190acdee29b5d20c5b3", + "type": "reasoning", + "content": [], + "encrypted_content": ( + "gAAAAABq0COvgomuqwo0Z3KFAStLOih2JmYOubPtUjtpD1zCRSCXdDnHGkj8asyVemaCsQr8szo_esDZ" + "PIT2Y9ghPTx1d5moHnS_lyX4nn3Almf4SYZ1pm8maTgEdntGFzcNtPRwW70-CgpeKe5z8zTwxsxyyKDb" + "OYTjt4hjUbAa4pM0K9JR9u5Q_P-LKgfcv3SNCeNSMs5lSV1aBMpuWEGJrlYF020C4wtmru3lH9VjQlOn" + "nW8uD1SCcPm7Sp1_Mm16PJBX4j2grbcIOJZDqj7OcjzRMAb2Gw96QfIg1_efR4BDVHzACkDClzPaiWuI" + "rAmGMA8bJDtQBPijZN3Hahu8eMoMd-XZDVQPx_7XM3F5OWUiuos_kEZNAhIIQHcLrXvyyf7slWvbT4Af" + "c8bvO45t1wOu5E_ALLhWcUCvbfPKBmHrlsRtBEVd_5n0q8jRwR1e5nyjPhMo3YsHvPmwHUJ5Gi1DZycD" + "-FDj6NpwlLcV3eeKaD8c-wNnHuXwfzi_KzIwU1phF_QxJ3ofegtM-3HwCqgaxxVFEhtIh3yezUKeqC7R" + "fSSHpmI-Q59TRf6-R7h6NFJKMG4Hg6C5twvVloHV2YMUHfBrgWr44X9FTzEwxJ89niDRtaB9Xvo2Qzme" + "9LUF-CWuGYTKj5QyRUQFOtclmBL335YTvTDWT8V-eN6KTE33XQNDEPSKuecES6fGHVCuIOeyQkWkvrW8" + "O2OrPJAmoW5g9AFWSWbeLWmDTTav6vbTyu0Ef9cvaRHRTOMq6q8VlEQoTQkJNUM8OKguY7Ad-1VSoIFW" + "D4_LvhuRsDsQiBu1diEq3hRNhXssBUd5wkFLOXw7UITRHONHhk7TDmVh0yjOwBr_e323vmL2e4BouKoX" + "x_0Umeb1jkjFlGgmpW59LmqXd91aXxjioYLxVMOMC7ib7O01_dD-OvWavSxAy9GLnex06mgThS2T5_u4" + "MRmpXIVVqdniqvoxx6YwX5F3TwRoIVc511oXyOG0QkKVLD1k0pI9OdFjiWNVH6XuhWPpVnAhqR6KYejn" + "hQQwuHA4KV4Nd6wOMFbhPjKPCuRcaN1vbgTbRJlwD8O2OV537hjDq6TQaezFJMk3HPK5bB_-J3kmUCul" + "YOXw0jgw-Ze8EmKLDZoCz3jq4NjNA9s0sLGd_Xg0kR6GeOmpLMWtT024yQ7gowGendkcHjIw3S-fIfiI" + "zJtZ-Hk9_iMBk8FE5Syb-j0kmzGF2gFj7ywTTdGoMsM3aTOvusVL14D1N_IfMryUdeoNJ41G8SvqGjtH" + "gyqCER9grGItYgKueDjt6IC5I086__HogkmrpzLZxSIpFCXX6rArVDsIYdS9XFgLW1oBKsVUs5DLnIYe" + "Ncv_cC-oTvEDBaYdhDPYbn_5N6q0q7dtbvohO-XJxOfLh0jC3TelEn7m5EXMPpAygQP2sPtI4_SmhA1C" + "Tji-B-6J60kBUiKmp65ZVxx8BDMI9dv3PLxsw4R0zh6fP_Fsok0UKJOQlfC6k47yhDuTcdN_vq7sAc3q" + "6P9b7Ud7Bz3JtY8Lg516vLK2vti4iwu7dcHlNiIoNZTGMANKYccO-XSRuJbX09bypgJB9sSR56_-Cb1i" + "b1C5RtkTv3ueRvq58Hw1RIINmiTTJn2MiYi_P-yXAirR39YTxcyRNUk4UDgK1lK73lrnpiBELJe-CPQg" + "wgLlalzjk8j0CnIMUA==" + ), + "summary": [], + }, + { + "id": "msg_03952e888f4f1827006ad023ae6e608190afabb95cbcebf3c9", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": ( + "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " + "non-integer quantities.\n" + ), + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_03952e888f4f1827006ad023aeb68881909fd9d88d37ef120a", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_mWfqifVeCXVWLiiNM1Ac2MjL", + "name": "read_file", + }, + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": False, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": False, + }, + ], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 154, + "output_tokens_details": {"reasoning_tokens": 89}, + "total_tokens": 466, + }, + "user": None, + "metadata": {}, +} diff --git a/tests/integration/translation/templates/bedrock_converse.py b/tests/integration/translation/templates/bedrock_converse.py new file mode 100644 index 00000000000..7051b86ded8 --- /dev/null +++ b/tests/integration/translation/templates/bedrock_converse.py @@ -0,0 +1,159 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQS9AUKEggSEAEYAjgBQgh0aGlua2luZxIMQHB9X3K/kC6GNasiGgywUP8bZriTtiro/2MiMAi/+5c3bmdPrprOsDzKxvo/J9NAn2cBfonQ" + "awxkQnqB3W5W5gKuBzPky9UZ+zTK6iqPBQhsFXO/q0Ky2I4cK8AZg1lsxNO6YTzWgD7fyCaEhn4HqY4hm4vK2V88DlytrsSGUTIQ/iEUSGVd" + "83vAHJy9AnRW9pPrgF/nW1D3pSD/Z3xeuMmxjMLuhGTF1JclSqe1onncl6i1Hu3a4fdTxXCKwAkgXjSPGMcYEANs9ziU1Ep+g/lPs4yIddit" + "mXaDnDzcCAq/6bFBnIe5IuN5nPeM4MqnXIXAhpDXa2R7HxUAg0v2dbzEzssSPuvn/PPF5VmusAXMQxM8zrLHdm5zUW6pss9p/OzxvgF6OlSj" + "8iF8YlOQXOa8avjTVUXiOLG4L67euf2YehPWDuWhrv1RsFJ3eDFvrlSESRnXeXIp7U4qqav5G7dJwxnFszrQzN/Oc/VgFvwErqp39goRlZiT" + "T2WpvuCOO8i8XsEg3NZUpTaypy2YfgMWJwVm/V06erqg3VKsVvIhXhp/XVkzu9AIHuPd+pJ1TgmybB4OGMuqYaj1Cs4OuNsH66YKEXSGexjB" + "wbjgPFHBXLbG/Ek/ATXpUShR8Uxu6japXknWqT19rusAv/7KRiqDZR2WiYf/nFLF7zMM6Zjoi/SV1AVC4RS5alTEsom8Pp8w8ZoptXCalOXI" + "fCh6kfqOZHxF9FCQ1i5hcPkvG2d1yzMkcV+i3worR+gXt/xZGT/Rdr28xceL9rOCEQawYT9fugI+BLHGEe/+X+jSAoFZxgcJ8Lqjz2kyoLqk" + "ZTRgYzIEBZgqa+dFrCVuXO9m1B3MSv0+f8gBeZVZKt5ItBWzIfOs1q8KyfqNNRYg3yQAmiWJsF0SkbJZl/MDQXzSXqkzriwclWGnRv0hBC7W" + "u75PM3a8YuBlc7Ww8Vdrw0AaG1s5FX7nhkHiV6DRII0YAQ==" +) + +BEDROCK_CONVERSE_REQUEST: Final[Mapping[str, JsonValue]] = { + "messages": [ + { + "role": "user", + "content": [{"text": "The checkout rejects quantities of zero. Find the validation code and its tests."}], + }, + { + "role": "assistant", + "content": [ + {"text": "I will locate the quantity validation and its tests."}, + { + "toolUse": { + "input": {"query": "quantity", "paths": ["shop/", "tests/"]}, + "name": "search_files", + "toolUseId": "toolu_synthetic_search_01", + }, + }, + ], + }, + { + "role": "user", + "content": [ + { + "toolResult": { + "content": [ + { + "text": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": ' + '"def validate_quantity(quantity):"}, {"path": ' + '"tests/test_validation.py", "line": 7, "text": "def ' + 'test_zero_quantity():"}]}', + }, + ], + "toolUseId": "toolu_synthetic_search_01", + }, + }, + { + "text": "Read the validation file first so we can understand how zero and negative quantities " + "are handled.", + }, + {"cachePoint": {"type": "default", "ttl": "5m"}}, + ], + }, + ], + "inferenceConfig": {"maxTokens": 1024}, + "additionalModelRequestFields": { + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "high"}, + }, + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], + "system": [ + { + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect.", + }, + ], + "toolConfig": { + "tools": [ + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + }, + }, + "name": "search_files", + "description": "Search source and test files for a literal term.", + }, + }, + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + }, + }, + "name": "read_file", + "description": "Read a range of lines from a project file.", + }, + }, + ], + }, +} + +BEDROCK_CONVERSE_RESPONSE: Final[Mapping[str, JsonValue]] = { + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 96}}}, + "metrics": {"latencyMs": 3022}, + "output": { + "message": { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + "text": "I need to figure out which file to read for validation, then use a read-only " + "tool to look at it—I'll check the first 60 lines or so since I don't know " + "its exact length.\n\n", + }, + }, + }, + { + "text": "I'll read `shop/validation.py` next, starting with the `validate_quantity` function " + "found at line 18.", + }, + { + "toolUse": { + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "name": "read_file", + "toolUseId": "tooluse_qWzJMc6h58NlspiEa01FUC", + "type": "tool_use", + }, + }, + ], + "role": "assistant", + }, + }, + "stopReason": "tool_use", + "usage": { + "cacheDetails": [{"inputTokens": 856, "ttl": "5m"}], + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 856, + "cacheWriteInputTokens": 856, + "inputTokens": 4, + "outputTokens": 230, + "serverToolUsage": {}, + "totalTokens": 1090, + }, +} diff --git a/tests/integration/translation/templates/bedrock_invoke.py b/tests/integration/translation/templates/bedrock_invoke.py new file mode 100644 index 00000000000..ff91325665f --- /dev/null +++ b/tests/integration/translation/templates/bedrock_invoke.py @@ -0,0 +1,144 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQShwcKEggSEAEYAjgBQgh0aGlua2luZxIMtEbP1UTgPIIXYE/jGgzmQCWS7ivK08RdGfwiMLQLZcccUrbxiBMQBmSloh21MAnvornAsyEF" + "qdKjYweOzk9gu7R5cyTNk5NT0z00kSqiBmieg1Sbhy23Upls1PPy2fC8794rus5qb4NKBmFvnYPn3255F5xzBh4tucc9xoUcjHkA9Q9SwGEK" + "VlBdgSfoIK5gvUMVz/VRPsvoBPfUJbbvEDQ9b2qhauTLOn/zxOwYfUAfKPPQQ/pQGf1Z5+n3GjbvzqcQll8uHra3BZyaJMmWZ2nzUbJTFP6F" + "Jkw95ArvOiyKeBZ65EVLskXAe2ZICsTd1pLESIfxuyeV4Yogt7drR5ytyOn/u1cM67Q0EWuZUQoU3OerB+K3PNxUp74ckJeK4M07IqAf3Bro" + "O7+oN5zWv7w/JfbtX9kDTSLRIu3eT/fmyGPrpgho2YIqcD4CO5pckkcLqhdi58RQ+Jnto9xZecoEdloi7sY6s4NYsa07X1qnOEQW0loS8Vs5" + "kcTpR+kl+KQJpk8pTf9GEqmtQGRo83KE4HKsHnd9pkdpex7thJIpxtgqR8OFY1Urjbj5YKY5mIO3FrsR2PtYK2nxgEzo3t0Ut74xj8ssIR7b" + "PoYxD66+mAZ10WODU4+o7uCm12FbRK/fPeV1uIfLHC5yUC4hw5Wi7mVdaEXjCPc0cwVnUFKpTKzVfH1fL3zNUermhufeqnCoLGhEOzQ5KJdX" + "Be9jAmOPd/Si5O5aZURC+dp8NXrVR1yDKhsWFg3CRi8DZ4wvgKNkmW+XYVB6G99gjHk7GmzcMSy5sxCWFs3qdKRbFyJ97WleW5ljuCz2YyFW" + "uDu3QYqWPOu0UlgzQcf+TDsLML4T8O6ddMqrErhmk52N1HW8rDHlEedB6+LlN2C+fcSEdLzFAdmmmQ8Y/HBmAzSzvsS9CYldAe9MmMKAr+J2" + "XBQBjJfbTvCYptNDuvqIChTBan1FII1w6Xq+NqaPR4f+16LM3HYKtIvwULH05YBRuZxF7B5J3NIKQBb39Rkhv2D/PTaMmdNIrDJqd57AKxSd" + "EKuGc9BFdyzwFz+X/3k/kO0Z6VZeLLIN5vdiz/9Aj1eYa3CnjVLiGH6zVUnhkx6eetUuHRZnrBx0ZXzRhZWf88bAQHgH9LwoTo61cf2Zqv4a" + "H5GvCCyi2gyhwWmBQ/7DeTUYAQ==" +) + +BEDROCK_INVOKE_REQUEST: Final[Mapping[str, JsonValue]] = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + }, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "I will locate the quantity validation and its tests."}, + { + "type": "tool_use", + "id": "toolu_synthetic_search_01", + "name": "search_files", + "input": {"query": "quantity", "paths": ["shop/", "tests/"]}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_synthetic_search_01", + "content": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", "line": 7, ' + '"text": "def test_zero_quantity():"}]}', + }, + { + "type": "text", + "text": "Read the validation file first so we can understand how zero and negative quantities " + "are handled.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + }, + ], + }, + ], + "max_tokens": 1024, + "tools": [ + { + "name": "search_files", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "type": "custom", + "description": "Search source and test files for a literal term.", + }, + { + "name": "read_file", + "input_schema": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "type": "custom", + "description": "Read a range of lines from a project file.", + }, + ], + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "high"}, + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect.", + }, + ], + "anthropic_version": "bedrock-2023-05-31", +} + +BEDROCK_INVOKE_RESPONSE: Final[Mapping[str, JsonValue]] = { + "model": "claude-haiku-5-5", + "id": "msg_bdrk_pobluozqaykrnd3exk5nnrtwrxipiquzg7hvvispjxn6qi4sz5vq", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "I should look at shop/validation.py, focusing on how it handles zero, negative, and " + "non-integer inputs before deciding what to read next. I'll check the first 60 lines or so " + "since I don't know the file's full length yet.\n\n", + "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I'll read `shop/validation.py` around the `validate_quantity` definition at line 18. I'm " + "checking how it treats zero, negative, and non-integer values.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_01Hfy4YfUU7hZUxEmpp4oDYV", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + ], + "container": None, + "stop_reason": "tool_use", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 295, + "output_tokens_details": {"thinking_tokens": 143}, + "service_tier": "standard", + }, +} diff --git a/tests/integration/translation/templates/bedrock_mantle.py b/tests/integration/translation/templates/bedrock_mantle.py new file mode 100644 index 00000000000..f046a1e0387 --- /dev/null +++ b/tests/integration/translation/templates/bedrock_mantle.py @@ -0,0 +1,335 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSkAQKEggSEAEYAjgBQgh0aGlua2luZxIMAi8mku8BopClY7wPGgwlWfmp5AbEIPA1xuAiMIbNUAxSlU56KwYLJ1P+/QAOb0fy2iNuUIvN" + "uu0NeTLet5Bf5yDruo7S0IwCi/HD+SqrA68zipbpxKWBeB+abRfM6xMZbYGcvRzePKzfZ6CYii1l08tO3fgsqbBitmj/6jvyogZ/a4Zf138x" + "ReoliJfLLudbpQB3rol3BtDrm8YrYqWW/vBn/Ioe3v3wHYRA3r4TllD0tyJwB4eNXENsq8vmxHrlTjQohAVsIyEjNy6xqs/YPW2zrCV85ZTE" + "UQQ2MDoCmAJa/wPgR/i68XIQ3kSWP3SllvhwlGcEUFa+lNUofPIUL7YBPyweBgrBSoS6hGr7LT3XZWNddtvjx6n5n1DF42FpapUdsK2iwerN" + "Gg/c6AHxr44urpLJ3XWjc/jgGs3KrLtfSFJ4pV3oRlZDn9upJKjTl97f6aIJqzxqJXlNIJ8VAWEV6kH/TVkRq5AdlwGQROLEA2taVBP01wAO" + "nxfwTfxj3UG49lVrwgi/+m3kEXOGsP7V876Ieb7wH/2zu+XxCz+k99d3bcc3MZr79VLxXif5QWbt70cm+2NjtCMPDYf0hm5o0KAtz9S9ZBUj" + "528KMDGsVR/rqwWcEV3mq/hWKjZmmeMSbhinrNVng5uM9cBe9YwaulbpS3s2lvQYAQ==" +) + +BEDROCK_MANTLE_ANTHROPIC_MESSAGES_REQUEST: Final[Mapping[str, JsonValue]] = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + }, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "I will locate the quantity validation and its tests."}, + { + "type": "tool_use", + "id": "toolu_synthetic_search_01", + "name": "search_files", + "input": {"query": "quantity", "paths": ["shop/", "tests/"]}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_synthetic_search_01", + "content": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", "line": 7, ' + '"text": "def test_zero_quantity():"}]}', + }, + { + "type": "text", + "text": "Read the validation file first so we can understand how zero and negative quantities " + "are handled.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + }, + ], + }, + ], + "max_tokens": 1024, + "tools": [ + { + "name": "search_files", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "type": "custom", + "description": "Search source and test files for a literal term.", + }, + { + "name": "read_file", + "input_schema": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "type": "custom", + "description": "Read a range of lines from a project file.", + }, + ], + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "high"}, + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect.", + }, + ], + "anthropic_version": "bedrock-2023-05-31", + "model": "anthropic.claude-opus-5-5", +} + +BEDROCK_MANTLE_ANTHROPIC_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = { + "model": "claude-opus-5-5", + "id": "msg_bdrk_cs3sdsy2tpe6vxiv7ipubz5hhgknryzkga3mhragbuieyoskse2q", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "I should check the validation file first, starting around line 18.\n\n", + "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "Next I'll read `shop/validation.py` around `validate_quantity` (line 18) to see how it handles " + "zero, negative and non-integer values.", + }, + { + "type": "tool_use", + "id": "toolu_bdrk_018J8SJwAgjhRkkmtm3xdJx2", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 50}, + }, + ], + "container": None, + "stop_reason": "tool_use", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 166, + "output_tokens_details": {"thinking_tokens": 19}, + "service_tier": "standard", + }, +} + + +BEDROCK_MANTLE_OPENAI_RESPONSES_REQUEST: Final[Mapping[str, JsonValue]] = { + "model": "openai.gpt-6.1-sol", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + }, + ], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": [ + { + "type": "input_text", + "text": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", "line": 7, ' + '"text": "def test_zero_quantity():"}]}', + }, + ], + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Read the validation file first so we can understand how zero and negative quantities " + "are handled.", + }, + ], + }, + ], + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. Explain " + "which file you need next, then call a read-only tool. Never change files. Use the chat_completions " + "client for this project review. Consider zero, negative and non-integer quantities before choosing the " + "next file to inspect.", + "max_output_tokens": 1024, + "reasoning": {"effort": "high"}, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "description": "Search source and test files for a literal term.", + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "description": "Read a range of lines from a project file.", + }, + ], +} + +BEDROCK_MANTLE_OPENAI_RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1792027271, + "created_at": 1792027269, + "error": None, + "frequency_penalty": 0.0, + "id": "resp_izakuft4jrxwmodefqee2jjt6gkmayywwuuc242pduoydywb3v4a", + "incomplete_details": None, + "instructions": "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. Explain " + "which file you need next, then call a read-only tool. Never change files. Use the chat_completions " + "client for this project review. Consider zero, negative and non-integer quantities before choosing the " + "next file to inspect.", + "max_output_tokens": 1024, + "max_tool_calls": None, + "metadata": {}, + "model": "openai.gpt-6.1-sol", + "object": "response", + "output": [ + { + "content": [], + "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJ7rFK0vbfOLCW3lZUKZX7nZu5wsyKm4u2DeoLRbILW0AF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBN1N2LzZrWDd1QWV1WkpjQ1dEOFQvSDR2Y0ZaY2ZWemtpR2ZWNGRncEgwZmJqMVBNd3FIV2NnbFlTcHVOcVMyZWc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcmpoCBv5IPHctwun5UTXrXrEmFVDe7MUULd6PZATyexPQO0PepR8jRQ3zVYSjkLkI8eYSjtPsT9kCKrIpsJlAuid+IimJvnTBHGplsaTC/53IpOSzB87nJlx+m4kCAAAQANtzzELZF5mKNBTvW/ca/WZ6iJI42OwsDigg6J5ZOIdJaC35TDfpQsssoDFQIHtI/v////8AAAABAAAAAAAAAAAAAAABAAABSyq/hcdJ8rImbQvE2c8qmEFM3skJI4sMoBBIdIZm8ljKX6zuu9kBGRln5StGWzlUOVYemoWeHBoDDuZkGDWAeTcQYdNeiCuTOQBg1kfIvhSGpGZHyKIfg+XnWIjrExKbZhmAC/GMebMNNuJ1/ZUFnq0UtD8X1r2xEkIWCUsQHysv9E6D6AstjrAgpo5arvZG0Gi5gryi/eE9Q7IGwilIvzeslUlM7sVcmD5RWe83nMeXmCUR41N50TK11F1j/H314CtKbJbMQWNWGLUUSRiKjqDFiC+o4bioQCRzvka6NUkIBitgjjkJ+etpL5ibB065xADWxaRc6WoUe1VkNVKXlYq4kOBNhnkJSHuasIJiD8/suYWXshD9XMF+SffKyKk3zscZwAuZ7N/ZOayYqGrhOMryCFYXvSo7v8PJ1hvg64z1MknImFG/6rSEVox9yeyfSEQmeDX/ABwO8PAsAGcwZQIwIpngWR/1wARV01RTUI+B4dV7XaR1AC0aOv5KfuQw5UJOnfGVp8E7MS379SPiA+SEAjEA0TMJ1LJ3cXitGqWvPATglnscU7UW9Mhny4elQ+cw7pFm5kRlYA/lkAyDdNyjZFVU", + "id": "rs_7a90cfc98ff653aeb165c2b845bc1e4b", + "summary": [], + "type": "reasoning", + }, + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to see how it handles zero, negative, and " + "non-integer quantities before inspecting the tests.", + "type": "output_text", + }, + ], + "id": "msg_4aeb32a9101256fe89e762125d4ffeee", + "phase": "commentary", + "role": "assistant", + "status": "completed", + "type": "message", + }, + { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_399ba0e7ac865a798b46a6cabe6b6bde", + "id": "fc_db8b798957b65c379754d08da376417f", + "name": "read_file", + "status": "completed", + "type": "function_call", + }, + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "in_memory", + "reasoning": {"summary": None, "effort": "high", "context": "all_turns"}, + "safety_identifier": None, + "service_tier": "default", + "status": "completed", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tools": [ + { + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + }, + { + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + "type": "function", + }, + ], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 312, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 108, + "output_tokens_details": {"reasoning_tokens": 39}, + "total_tokens": 420, + }, + "user": None, + "moderation": None, +} diff --git a/tests/integration/translation/templates/gemini.py b/tests/integration/translation/templates/gemini.py new file mode 100644 index 00000000000..d699cb833e6 --- /dev/null +++ b/tests/integration/translation/templates/gemini.py @@ -0,0 +1,257 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +GEMINI_GENERATE_CONTENT_REQUEST: Final[Mapping[str, JsonValue]] = { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + }, + ], + }, + { + "role": "model", + "parts": [ + { + "text": "I will locate the quantity validation and its tests.", + }, + { + "function_call": { + "name": "search_files", + "args": { + "query": "quantity", + "paths": [ + "shop/", + "tests/", + ], + }, + "id": "toolu_synthetic_search_01", + }, + "thoughtSignature": "c2tpcF90aG91Z2h0X3NpZ25hdHVyZV92YWxpZGF0b3I=", + }, + ], + }, + { + "role": "user", + "parts": [ + { + "function_response": { + "name": "search_files", + "response": { + "matches": [ + { + "path": "shop/validation.py", + "line": 18, + "text": "def validate_quantity(quantity):", + }, + { + "path": "tests/test_validation.py", + "line": 7, + "text": "def test_zero_quantity():", + }, + ], + }, + "id": "toolu_synthetic_search_01", + }, + }, + ], + }, + { + "role": "user", + "parts": [ + { + "text": ( + "Read the validation file first so we can understand how zero and negative quantities are handled" + "." + ), + }, + ], + }, + ], + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. E" + "xplain which file you need next, then call a read-only tool. Never change files. Use the chat_co" + "mpletions client for this project review. Consider zero, negative and non-integer quantities bef" + "ore choosing the next file to inspect." + ), + }, + ], + }, + "tools": [ + { + "function_declarations": [ + { + "name": "search_files", + "description": "Search source and test files for a literal term.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + }, + "paths": { + "type": "array", + "items": { + "type": "string", + }, + }, + }, + "required": [ + "query", + "paths", + ], + }, + }, + { + "name": "read_file", + "description": "Read a range of lines from a project file.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + }, + "start_line": { + "type": "integer", + "minimum": 1, + }, + "end_line": { + "type": "integer", + "minimum": 1, + }, + }, + "required": [ + "path", + "start_line", + "end_line", + ], + }, + }, + ], + }, + ], + "generationConfig": { + "max_output_tokens": 1024, + "thinkingConfig": { + "thinkingLevel": "high", + "includeThoughts": True, + }, + }, +} + +GEMINI_3_8_FLASH_THOUGHT: Final = ( + "**My Thought Process: Initializing Validation File Inspection**\n\n" + "Okay, let's break this down. The user's goal is to help maintain a Python project, but important" + "ly, to *read* relevant files and *never* change them. The instructions explicitly direct me to u" + "nderstand how zero and negative quantities are handled, and to *consider* these before choosing " + "the *next* file. The user's explicit request to read the validation file first immediately trigg" + "ers the appropriate response.\n\n" + "Based on the instructions and the user's initial prompt, I need to focus on `shop/validation.py`" + " which I know is at line 18 (`def validate_quantity(quantity):`). I need to check how the code h" + "andles zero, negative, and potentially non-integer quantities. \n\n" + "Before proceeding, it's essential to confirm the instructions: read the file, explain the choice" + ", and use a read-only tool. I need to be precise. To avoid any ambiguity, I'll start by reading " + "lines 1 to 50 of `shop/validation.py`. This range should allow me to see the context of the `val" + "idate_quantity` function.\n\nNow, for the critical step: I'm going to follow the instructions and " + "explain my reasoning, then I'll use the read-only tool.\n\nTherefore, my next step is:\n\n" + "We want to see how `validate_quantity` verifies quantity inputs, specifically whether it properl" + "y handles zero, negative values, and non-integer types (e.g. floats, strings, or None), before w" + "e look at the test suite. Therefore, the next file we need to inspect is `shop/validation.py`.\n\n" + "\n" +) + +GEMINI_3_8_FLASH_THOUGHT_SIGNATURE: Final = ( + "EoATCv0SAWkUfRPAfPpnzi1FIwmV+NEV1p2Nir3bkvyVz8J7BJ5g940gYIMwrVGMcoDSKUSW4RlG2+bV5u02G0uwFXQsxVCE" + "04rAU5hV/Viy6qJE2RBz6dj2/xUR2klCfwzqf10KF3Zm65efbvNAYTFQuJmLvuPNgavTk57xuBSyRE1iafGbfjkpKyE2ASVM" + "uq/70iQ1tx6XusTyMLAFkFkIg4KYsEgZcrUFZaAsI29vgDIRCKw7ZBA29INSlhZTvEn+ZeQd4Oi3jI983paxvLHewUv1Sb4j" + "PcrZ7ULXhZT8/CNGvTTax4AvqbaAOdPcKJH+kI+oG+yvMhF/nQjrR+qnUB3dhL0mjVcg9QQZh2Z0qZpZH2tsSCl2pqBBdcaU" + "V6C/5nLPktRkMA2PCE63AsMUY+yfbhnY5pGr9s+C62QNl7Tofuc7FrH0UDNZqxb0PNrShTgv0/MpaKbwS6EsYb7rl3NHZuD2" + "Tspu1n9HjhD/imlMTbJXldIko5khSt42o0z8mntYiWAj76TbtKTiELNC6Vbz3YX9jQu/6QXiHgIwZ32pfAUhOH6XYr5mHFRo" + "rsv3lFq7WyZe23j7bS03hn+hzqjOgeGEHUCmOHisL+pRajXXKA/BqrPTbRR92l603tIvFFUjIdmXN5Yn/086YDpcvOYzO9UC" + "zQcHg5vfyy77fpWjTKVxd4iE9PCi/RqqPsfplW0YgiE7635HnyOBUF8v8KNCNeU4BJRjRc1lnZKFHN3mv3t49UWgdPqxIQMp" + "CHlyKL96+88Ot6Sy+r7sRVrttrE4yTWn72IeOBd6N1E+vX427hCcw3MQBaxTaSJcs7h/mJV63jvGAVEQxWOxWRE2lAEJed0l" + "ttQS/rv9V6VU5CDVKR3zDcDE2uLgct7KTDHB1v/jjh5sZqDBZURLEywUP4HAlI14FeFK9ma6ZG6sjp+KVU6EkNQingdAH+zh" + "Nsb2+MkZ3Beccqlf9pEnmiMtT3xyQT8yQfXLL5o7GLIkFLemfe98lvThNiEhZ9udldYZgWi0g1sQcjrxcNu6dvH+O8v9RWvD" + "A87rM2rikA/oJc0IBRDvFNehMtnu3yR5969Q13ORotFWvYLOwAdSYP5H8TIvpEleHQsYGnPUclZqTI51+dbydYBXnqTDtRer" + "7zq4Ul/cuF3LYLRcr4zfkNrOgZK+U3Rfk60WPEi5AyWQ2IO74B7FQqIHKw5mXjlV0Krfn4e2bYJXBRS82xgfK6pYtHWgIbY/" + "18quckh1T4bF9JfRi0JTPYp2sU2FossIjbT98rpd76OBHBnWl2BUcd+0BiZ91DNErQVVaAmsfGA4li2upU0E0QnN87tctY43" + "DLL9YH133uCTV8YQh8hfbh4j30GcongmSd5L30stAv+AGQRFqK8fi/vJdgTXZXWWYSjTlhetY5ar3FnaulztywTqjLbBUxK1" + "jicit6aonk217FYXDzSRy3BcGx5TN879JkvQ22zpsxy41Brxw8MhYT92UduPTXcQwNkh09+Rn7+lP8/u1ppEtE/vttmHGuxe" + "QEyOk8isAtoErC3xdA2fYh16dA2on1HPXlX81/dF4Og1PQdZp3GIS3ftuKMlinGuRyFBaHV5IZYji16r+afCWnz3dh9eKnrs" + "OrdjxyLk1wcsQRzig6nurhBkoOi1p73QyzVyx2hzMkKoNAkV2YM/33kPrwM8mXQYEpQVL0r6P3wgXFP4ep5dYQ+HGZCan5i4" + "QWum9+1FOFAIFUbxp5ePAybhzOhLkOn+2EPwKumY7hR56sVsAyJFp5cy7JlaxPfI/n1Ez9xyOlWimU6mTrzM9VW7J9XvEPGe" + "Nf9DRRMWSTopkGQSwdRVzcoNv2DyGi/7p+pXoI7iEZyvVwe/ascmsE1J/GKnJ8niLmCIJCT0oVyZCnEQIDuCzZ/CBYePfbhx" + "dNFrZWka7k3/mc2+sFo83qatJE3f0twjTwVzrDg6AQ896S2JlGASIpjTkJVo07H2mSyQJheQnA3Y2OpAnWWBtbmDbB0kH/fF" + "WISHelg4DopTdf3WyM18ONMgTiFCH+U7w2qe+PV10bD0jED6P/mN7LMvYr4sNRoBKekpnS71wNzH0J7SPzNV0dVGAei34EeX" + "wOOXivap9YTwtZGCtCVMoeOr+0gsdQBeqjSyLeViDH+FEBWvteKQI7WRB3gHLjGPnKpdxghiqfaZsZ1jD1xCVAAIMpZR8h4L" + "y4IrIz0WHJ6ZCECglEty/iSD+lz+UJmBxKm7T8aaqv3i8wpjlVhM3rpcmoWN9E/ESnS+mkQfFaqjy+uPHUHP0m+bG2xJ/AN9" + "+XEi4eE4mw3dSIlVynXsa7I2iEyAqO8B3+KBmvvQ01aSl1a2kITFf46FdFaBN65gn/k/GlgJAa16BOCARj+uIoKD3GgvetC6" + "IOb0F4j+WGS79UMwXWJ54twqKT4AhBLjoq22yGflFRfBOrkby5iSmMjKanMiFM50FWltbmKm46lJLMzXt+fuZU/7/QRfDoku" + "a8xW4no8mY1nyvpoPqkyf4iQbGkSuJX6kgsUi/Oc5UsYTgfVYA5HJH0BOBuoFZqeTUUY61lUysFnVFJfZRbXL5rbN3kCmP76" + "Ed/k2vb7Tzgs1dtnKjbA1p7I1LQ39lYg6YDKSrTPzr2JtM9RGHkfebesNEYSRSUbpDXpvsVBpBc0tNK0Kf4PEjbcwN+XlPMC" + "8nd8PlMtwM6QQ7Hzd7amukS13rBNFsyTnViUXqDEmbgbpw7A+KJRmixsFYUwGJlramZ2MZt68kXwpG15S4kluZapEBh8HrCT" + "0aUj11aRCPvc78Ojnw8qhOTYX2to3HIzj4FCWN5PQ7Tk7jdaD0rFxoRqa51wIhQjs9DuG2b+6nNpPrlDJX6w6fhA7C3hGJHV" + "kq/vWzgfC2kpVfhYd6KC+ySTTkCSR7JtBpAztwfU0iWaBnqmU7ZdOVidW+bX/DT+yx7An+1ugxf4o/XTWTyY9+YOCrgwKccB" + "S/Md8hYcUvKnSw+WYQZtZTT4QgFLooXeG9epQldH5L9/8dC/tyhdf9aepP7tDe2ENd6BAlv2NkXf4UFDwqA79G4MJRJochrF" + "8WE5UgHy7kMboEgvNWcwKjNo7qv+Lsj+p8UnlJ61Mh+3LRq1l77VUEe/xPo1jrR5LhShIWGht2R1ZJGykwtP5GwZMkmqTkLN" + "uvyRFOKiWZf265aXW89HuHX5SREcOcp5YuFmQGn0vQroptxyIEk9fGZPVfIaoxGxcgP9vzgG43JCdRc=" +) + +GEMINI_GENERATE_CONTENT_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_8_FLASH_THOUGHT, + "thought": True, + }, + { + "text": ( + "To check how zero, negative, and non-integer quantities (such as floats, non-numeric values, or " + "values less than or equal to zero) are validated, I need to inspect `shop/validation.py` next." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "shop/validation.py", + "end_line": 50, + "start_line": 1, + }, + "id": "call_7668", + }, + "thoughtSignature": GEMINI_3_8_FLASH_THOUGHT_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + }, + ], + "usageMetadata": { + "promptTokenCount": 358, + "candidatesTokenCount": 80, + "totalTokenCount": 1013, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 358, + }, + ], + "thoughtsTokenCount": 575, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "1CbQat_JEK-jqtsPtf-QoQU", +} diff --git a/tests/integration/translation/templates/openai.py b/tests/integration/translation/templates/openai.py new file mode 100644 index 00000000000..7fef2f6d2b5 --- /dev/null +++ b/tests/integration/translation/templates/openai.py @@ -0,0 +1,371 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS: Final[Mapping[str, JsonValue]] = { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + } + ], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": [ + { + "type": "input_text", + "text": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": ' + '"def validate_quantity(quantity):"}, {"path": ' + '"tests/test_validation.py", "line": 7, "text": "def ' + 'test_zero_quantity():"}]}', + } + ], + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Read the validation file first so we can understand how zero and " + "negative quantities are handled.", + } + ], + }, + ], + "instructions": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the chat_completions client for this project review. " + "Consider zero, negative and non-integer quantities before choosing the next file " + "to inspect.", + "max_output_tokens": 1024, + "reasoning": {"effort": "high"}, + "stream": False, + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "description": "Search source and test files for a literal term.", + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "description": "Read a range of lines from a project file.", + }, + ], +} + +OPENAI_RESPONSES_REQUEST_FROM_MESSAGES: Final[Mapping[str, JsonValue]] = { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + } + ], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", ' + '"line": 7, "text": "def test_zero_quantity():"}]}', + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Read the validation file first so we can understand how zero and " + "negative quantities are handled.", + } + ], + }, + ], + "include": ["reasoning.encrypted_content"], + "instructions": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the messages client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect.", + "max_output_tokens": 1024, + "reasoning": {"effort": "high"}, + "tools": [ + { + "type": "function", + "name": "search_files", + "strict": False, + "description": "Search source and test files for a literal term.", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + { + "type": "function", + "name": "read_file", + "strict": False, + "description": "Read a range of lines from a project file.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + ], +} + +OPENAI_RESPONSES_REQUEST: Final[Mapping[str, JsonValue]] = { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + } + ], + }, + { + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", ' + '"line": 7, "text": "def test_zero_quantity():"}]}', + }, + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Read the validation file first so we can understand how zero and " + "negative quantities are handled.", + } + ], + }, + ], + "instructions": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect.", + "max_output_tokens": 1024, + "reasoning": {"effort": "high"}, + "tools": [ + { + "type": "function", + "name": "search_files", + "description": "Search source and test files for a literal term.", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + { + "type": "function", + "name": "read_file", + "description": "Read a range of lines from a project file.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + ], +} + +OPENAI_RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { + "id": "resp_0207c20ae125feee006ad023aada5c87d082ce8ba652474312", + "object": "response", + "created_at": 1792025514, + "status": "completed", + "access_programs": None, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1792025519, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect.", + "max_output_tokens": 1024, + "max_tool_calls": None, + "moderation": None, + "output": [ + { + "id": "rs_0207c20ae125feee006ad023abed4087d09d32a32ad2d3182c", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABq0COvx3R_lOOaYJijiIokUKBq9tt_p3mhoPHnm4KR0Pj_t0wf1sP7WEDHfYSzylhHilYoKjU4guIz-TAqBV6HNUpIx5ggp3aukKEW6zks2ERT3a40sCFFdUdKHk6py5trFuwYn6lHLwifwDkd7rI7MKIP7C7pZSHnXh8cmfk0t96p4nBplQ93C4m917Kyx6PqvWSxPP78h5s_RMEdhkWRU20xMhminpIBNArOkPYT4xsmBEuONlndM5-DHG7ZeLk2yMZw8b_hxk2LZ4mC66b5RHfJKVKfw4egn4K8tkr8mQSR4bnjpY56DsIRj4fjSH5GpoDZyolFFBh1BNkGMoVsicye_4gZn0eELuOWxCw7lVVsMpIzTLw8CHJkCo_8vKtNCFSZcKjOPKDDfX1bsFJ7LMonmqijHPV_kZi_YXjVNNJBhJLiGjYrVUFZzjS0OKQx-RnZf5luO3UM2Ff3Ilvx1eQtuWlsvJjPBRwObzk5zlPEdYwoh5OjUREbgGb4H71Q7A9YYjUIms2h5e13Z5s8X0onRVr9SUcSbbMA0FXWn6QGQ5qgiQV56xec7xk62SrxKBXeILgBttBf68YfLAbdINCPL89G5Z1X2toQLlFM7e0jKYUOvSmpWyPLZ3q1040oDf2Q92v6NjOwk4WlLtZF_4xrZ7bEIWmea7zcDEnsvNBRdKaUbeUOm0SfRuIH4gQfbbueSqM3i12eK4TQhTbJ9VivUUovUv7pCKvizdQuunf2MMXWhq79wLYQYXIleQ06E9hbx6QT-Ue5ELQVC3NSLhXiwucOSqd-NZFDQA5wgW2Uaq_-bFYzNOPwtArfaqnHOILmr5mDvWCrlnLs-DUPHU-DtzYH7Eu1VSlcEZLaZX4vC9ui3x2aOpTtmVCILZpw7MkouVxSR4rFE48doVoCamJHhey-TZ7OMl0A5JdacTt6oUE8Z2HEcuVh9yGAZe_Q-YB3p4egLbxO8ICWUkwSSNgrPq3WRbT9sbT-Ko7N7KDAfgyvzzdEYMKDmci6GBrVsAiscc97-gxbhVhlrMyIXmgTluW07o591YjlOkvERXeJobEfby-VUJCu0cHd0nMudzrhCpHIhU3aUMkk9GMlSNDPFnKeV-gylycLhYRb0xo1kXMRom82fKmvUgO5Wh_BhsCdeGaVQ2bm5wyLT4x9VhEe-uOwaYG02XxoSiIFqqSrf0J8auFHIRpUTKzNWMK_GcIaU6o3ON0wOLKaga3wcyrGkG6wtvGkNLCrvmDIXg0bmvFGZrmybQvys_u636NXVS-Y9irqUv2ghu5tyKI_F8uwibi9JkKa5XcotQGbAToTE8qaHfowZOdTyjIKrU3BjL_Hs5M41KXwrNkk_X5ttJxEPW1qCjyusFb9yeGa9wzHMsp_VO0ysdO8L2KSRAY4LHpGI175VxUmnSg66PtCPln0OYxWsX2FWE1A7B_la662tO4VoS3bcslVZreRWFDJ_7X8qKx2fStxd77B7JkG1ZElxlZYlBRN5_Lo_w1yM9SvqHzzewQYEprSA1gltjxAbIzy7xXReaT86PNI8ojSLfXOkXa6Cybu3hIxsIomkZwTtAEx85BoIDn2B8oTp744Lwkwn4ualAfV9TebiqOWCUYp47fX9rimte71X6Hnvf7tWEBZMMwSb0pTqY0boMOS0w3XKKyEVRNyCoGWMgyvhAxk9E5iao5N6i5W3aSel27QwsUaE4YDGKKMKSryH3BqP6X7d0kdUcAx-nd5yudVbePQyc-UC7EbISj5QXY8g6cTOy-IkwpBe68DvfcNGIhuaH6UscqrewXQjKZFJC_BJSl-ICOfMw-FCTumzCDXPqTeLBPHf1g5kd5lBd6HLPUldHtJfD48V12KCRtaoksZ4g2NibuClnmZRpnDMPDUr7r_D8-RfoLEiJeh8SbOSnLvMPVAJIB95F73B-UX5u0e0aWeyfG5QjBq-kB7vVGsRiMyHEVQW810xo2qULVIyVO03s7rAWwTxFY-EHmKkaAIbKU2b-qAasytni7f1Aq61OSIYqSCRdFhpUX6HpnQeCFC_s6_Q9P3CFx120ZaTB_HPeRJO6dhVSJ3nL3PiNi8sCnhRiNIXkJfm9l1TxRS8ok9s1NY8oi4Y98J4Ay52EkX6f9F9FtWHNu4GX_Z50jDNnVTVqQo3L3slpwxECsTAEU-0MsmIa-1RqJ00VqBLKtIDVbbnQ==", + "summary": [], + }, + { + "id": "msg_0207c20ae125feee006ad023ade23487d093e272f84cb35723", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I’ll read `shop/validation.py` first to check how it handles " + "zero, negative, and non-integer quantities.", + } + ], + "phase": "commentary", + "role": "assistant", + }, + { + "id": "fc_0207c20ae125feee006ad023ae893c87d08068561e36e665ad", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "call_id": "call_S4WtfrQwa6ZLEUZf1BVtA8ON", + "name": "read_file", + }, + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "high", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [ + { + "type": "function", + "description": "Search source and test files for a literal term.", + "name": "search_files", + "output_schema": None, + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}, "paths": {"type": "array", "items": {"type": "string"}}}, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": True, + }, + { + "type": "function", + "description": "Read a range of lines from a project file.", + "name": "read_file", + "output_schema": None, + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": True, + }, + ], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 309, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 118, + "output_tokens_details": {"reasoning_tokens": 53}, + "total_tokens": 427, + }, + "user": None, + "metadata": {}, +} diff --git a/tests/integration/translation/templates/vertex_ai.py b/tests/integration/translation/templates/vertex_ai.py new file mode 100644 index 00000000000..5ed4082d04a --- /dev/null +++ b/tests/integration/translation/templates/vertex_ai.py @@ -0,0 +1,281 @@ +from collections.abc import Mapping +from typing import Final + +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST +from pydantic import JsonValue + +VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_0: Final = ( + "CAQSsAcKEggSEAIYAjgBQgh0aGlua2luZxIM5AdGqr4T05oHxpggGgz1Xe5nFTdJd/ZREp8iMMOP/ne3HrVXil/Qby25EvgVLzla" + "PlAqV9EuGZhZueHwbwVDEAZISWNgb6af+m03KCrLBoxdew1rYrrcHnuY8IsIpAvjDHAWs/wAS1J8xCNJO8pAWdDUkXz9je0gdBkP" + "5EGORxwGsDr2ATCotY/rWi51A5rSI07lMVVEYjko0F1xbLOEW0Q/H76nWrp30p5v6HfxDmxBg3WvzYmu45onzgN/Q0R13KPzAy5A" + "4RWV3ANP+SRwJFQyTSFLqn0O1dNnd7eENp2TvBaV6ruF+qbrEM3Bdfc3XkyCSQMn6RSSNM+LkK6VrMdjOOtQZybFHgmOC+ksFqFJ" + "+nBeLOS0wI2QbaIVVR7k5yFGoGW3DLlIFDSKTzOPnhLJMdSMvLBpf156LuUkjk9cHzgriw8kyZJbVtuw/Myvwc5u2ZQsuAfuqwd8" + "f5Ewf6OPt609o+C0G9dzKlt7jV+ZwVfMIm+MbSO0nxVaFMgZwBFrqcLH/+ie5m2QCBjVJXMUcc4dy2u7NfYxN1UzLJjdHzp/EOox" + "2mieSdI0Llx9sqiWmJRlt73evYTY34Z0V29dFOaJGH6ZecuFj7JnaR4EhLTZZJixesoYKy6b8wJB0ZsLKsQ9N93Kbxc4ODzBZnad" + "Nbxx5JH/weFk5MDKBtIiCoBXusFAvDO5xO7G/iuO3Rq1MB9P1GFf7e+CB6Y5Ej/5yL1nCivCrEd4tYJUnl3DCjSW2QuV+JqwmclM" + "08RgVQKJ50OXYQNgev8aO1JSrs8NpmtodrUrgpwLDSqetMTsm8LkpqZadc+73gGjAscm4o9Q4+m02gpqLklXxgS+QxN8PBTNDC42" + "VJzXowlJJHRXoczxrt54cj/iY8VEcP3o0Y99zNIyvbvCyZypnrkTfYlEQo+1+c5tjcUiMGaXb26UZpr91VO6vYqS06lgLIWYjlf1" + "qF8Ym+EqPRVZsz/YZuy+7laG/9nF4qAIMWqASxL2BFpdPrhePcFrlyefEU3zt5Vl8xvR8K4zv9Glmtd+3mVIwDc6fSAwdm9rWtzg" + "92Mq8LzeyxrNcLijq8iWK1CnFK3ayPJgSicuEhXgD0JpgltQh+lWZ6a8M71+/SGyJJb8hQp5x5wpEKFgGKTCBvl/oV0Op8/Ie44I" + "IYHkm/aOvu+WSnHwU4IrcHwPBSUbk2pYKaWQb1cRlDfXWmuaHluOfiRPo0ORvzOdehgB" +) + +VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_1: Final = ( + "CAQS/QUKEwgSEAIYAjgBQgluYXJyYXRpb24SDCLtxRBlqq3WVMEpRxoM5UkHHwV72f1gxZ0HIjB4+DqdmR5Sfg3wso78vOXyUJMZ" + "H7w/XBD3JSH6I78plRQCkTuRWhd+umfXGpb9if4qlwXOtx5CQ1E+AabA3vqpLJarGaCFHO8HclGAk37uPKxmyKk7hSHvMyn3ZIG7" + "aVPOT7f42kI2Kif4/xhMrZ0jANrgbxsT2I+UfbFVXeOaKFjkG3Enn46IXL11N+4vP8RbdGESWDxcWDyLmp0O74W825FHNsrf4tNA" + "mfH19Mq1jXYC//8fsz01HC7mlsTGFQShBdbblbKcV43Mkdsq1OV1EOkUZf5JX1AKr1D2Ywsp11aoU//VfXb2Bq+/o8E/2ZRupoTL" + "d1Sak7H1aNdfJ+5mfDHGLOrlPjxunSxxksfrZopjVFs8fN/OsSX2FjCDeHskShGoAF/75pfDHvQl+vsZgWgprTZ7nZ2Sirx3Asui" + "670zquqc02ke2ZBxFhVvbfXikAm37QK3sSyzMp0xtXx7+B5TqguHMBPcZ2FwMkkejyM/2c8p1UnoimAjEnD7VxWuEgmg72cL62Qb" + "TN1W6qRf50CQnC7yrTh+23804bQ94kea4mobxRPcveyZVCtlcwgKcvFzWSqZ+QYVORHbnBkKIQlUZB81VT7NxM54UN2Lu2clNu3E" + "/6r+0pYrbmJazAtcaT1LLFS5wrfl6ngHGTDdPGIwQPZRclrL+tV3YGApuwauC1QQRD3NG8XAuHhEbtJcxOBmGx1ctOsHiRrhoFjM" + "MQ+/qNprXLGy/JWeSOdJdwpWOIuqKf8g1LH2NDxbMHOnkIN+UbQBBCbMDdV8oLbpu7n8+moK5ggx4sQ3sWcn9/aBl9VTUC9Qjk3/" + "GTNtZNGzlWBVn7w/JnYZINjy40DtReIHdqcZja5XKnYpzfh7G6EcYA7tW4SmtWb2sPB1Msz+S8FW944iQ+AyfZQqic9b8BEw2JYG" + "it0butjvG2EWi4LnL966v5WCZQAYAQ==" +) + +VERTEX_AI_GEMINI_THOUGHT_SIGNATURE_0: Final = ( + "AY89a1/K8QT9La7l3gDc5jImwnXSF0xUuyg8rAUnAyYS4c2/Yjd/fV3Ve1tHSww5bSooGMdZ0oqljadXXxUqYi+7fkKOWD0Ij80K" + "LFu1O//r13u8cIpIqUxEi3lNiEn8rK6WVW5DPTMTBddZBL+/DcpCEi7U3m5vGYfoJUm9AJwQB8XDRdqLdA1hfiRnCqGpLJsFAQ2g" + "Ln3FpQUPR3+FRoWRlJnNRjk8vWaq3mBKHSHttLMRd3rpvo6VAAZbs2OGoBJXFXA9S+RyN532kDK3fZIGGW2Fk+FSD9EJ1hN1+Qeh" + "v8gzcxyZOin2O/ONFWEQ4zf8lHTzaostuIMLkKFM3v/ssK+Y4VgS0sPFr7VFByhczwoDx73oh141iDD5ch7tqAgnR7YikrWQDuV2" + "8FBSB7+MQ3BrBSt6XUvYgcHsGNbg0giE3D3pFDR7uvLfvgjhFFdIqnCWWmPfBgZ7GOD55unG0IUPA2udP65xg88r8TjfpQOUsOnN" + "KZLOiKovzHrXZX2L0tRGha5nE8ICi427Mc5YF9lm0Ilpo/BrGuuusMG3Zx9RxRyjRa9+zWPANRrvuq7SOVnFvAss4uu9PYf5Orwn" + "nGFJ0xjP1hn22yXOxRYy62ySQQ1ValpWXjPPXvGKZRiUArCuvrBvDEZoUfJhlcVXs26kYnqRikRLev4GZGvBIDnGHyWLvMo0gx4r" + "1kBnAGl5WyfZJu5SFywm/o5fMP70zVUoVdY+zC885uqB5pEV+VsAMyy98NumfmBF7kGtIjbt4l+hAqpzM1ME1z0kRqN7n9JEKn0H" + "DlsQAD9V582O2bBVSTRzF8aFJ7RKohGhrFqyGmYLA+ZAv5Z4kJ1HBZg1LEz2G88MT2vznP6lCEH0m9da48Q5/85pNQ5dlGGEkhpP" + "I+xazh6EZ0cFNGdNY1vAQY5rBQ8LPwizNSKenYvqwMdobWy8xReIQWCsOk2l4QCtlu8kC+iabWrsCPOfjdQmYTlpI+emQwssWXYU" + "UQ+wDGl7QWoaSAnwGu+XQMQ9ndaUycPdBu5+cuw815uiNNtIVfeKejFIbLF66Jq/duKfZ9tgWthPzpoM50u9aie2FZwNOuEtaLIv" + "WcdtBpY9LvnAemSuDUTLLSuiQYwBwRq8HKdFARJ362Wf8gg+ldoeXEV9R6+cscPaeY5j/+E/rsTWyxixg614qiFQyF5xL/7DWjUI" + "LBuIrmJUIVcu8LelTUslepTGF6CBTJ0RABv8pK7S8dUDy+HXtvT8bmz8l9zOsaTagC1jUpvaeGpi44gorN9KU7z56FMxoKSeKF7M" + "B2zuTFBN/GekI9gddLfQRaVK/9LAD4j+uBzdt/XICe2kGqjNu3ds9tnx82/9beezRyTRfFQ+p1V/jfaEt8AMTZ5I2IZQAkgkAnXa" + "KieRKs8qKbkF5yjhtI0krsyg4+BDQZzog+dgkxqYxLDuNyd6hN9HlnMEBAwW8hxWGp4WE8GvL2IU529JVMaBvK/NcrGNnPW/m158" + "BwaIFSfbSxQbqjv6KKqMi30i2bowXBAJit2YQHb3VN1reFBynM9mRm71PGLAbizdVS5VIqNI5zGquZrgpmTqqMF/AjsWpVSTo6Gi" + "WD3O2PJWReehkH3aMJMFeu3jv0/Ru+Ihw1zwFcZr0Yh7rK2NEYQyZjcmf5/vX0rBh89HEVdc78j16arXcFWHNZS2lLAm0NC+Xmak" + "WIP3YUcba6POPYKwqP2huUccyyGg+qFPTcWF488PNm4H86YQRrnUSkPEofjMuaLMrp3vS1TgpjuUZRiZ" +) + + +VERTEX_AI_ANTHROPIC_MESSAGES_REQUEST: Final[Mapping[str, JsonValue]] = { + **ANTHROPIC_MESSAGES_REQUEST, + "anthropic_version": "vertex-2023-10-16", +} + + +VERTEX_AI_ANTHROPIC_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = { + "id": "msg_vrtx_011Cg38nogK8THpR3HzMQ1Mt", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": ( + "The user asked to read the validation file first, but since reading the tests is independent, I " + "think it's fine to read both files together rather than strictly sequencing them.\n\n" + ), + "signature": VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": ( + "I'll check `shop/validation.py` for how `validate_quantity` handles zero, negative, and non-integer " + "values, and review `tests/test_validation.py` for the expected behavior.\n\n" + ), + "signature": VERTEX_AI_ANTHROPIC_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01SYkCZM7zmQWqcyokJjiAyf", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + }, + { + "type": "tool_use", + "id": "toolu_vrtx_01UhxynEUZYnaNhRkRGC2ZGn", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 50}, + }, + ], + "container": None, + "stop_reason": "tool_use", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 425, + "output_tokens_details": {"thinking_tokens": 229}, + }, +} + + +VERTEX_AI_GEMINI_GENERATE_CONTENT_REQUEST: Final[Mapping[str, JsonValue]] = { + "contents": [ + { + "role": "user", + "parts": [{"text": "The checkout rejects quantities of zero. Find the validation code and its tests."}], + }, + { + "role": "model", + "parts": [ + {"text": "I will locate the quantity validation and its tests."}, + { + "function_call": { + "name": "search_files", + "args": {"query": "quantity", "paths": ["shop/", "tests/"]}, + "id": "toolu_synthetic_search_01", + }, + "thoughtSignature": "c2tpcF90aG91Z2h0X3NpZ25hdHVyZV92YWxpZGF0b3I=", + }, + ], + }, + { + "role": "user", + "parts": [ + { + "function_response": { + "name": "search_files", + "response": { + "matches": [ + { + "path": "shop/validation.py", + "line": 18, + "text": "def validate_quantity(quantity):", + }, + { + "path": "tests/test_validation.py", + "line": 7, + "text": "def test_zero_quantity():", + }, + ], + }, + "id": "toolu_synthetic_search_01", + }, + }, + ], + }, + { + "role": "user", + "parts": [ + { + "text": "Read the validation file first so we can understand how zero and negative quantities are handled.", + }, + ], + }, + ], + "system_instruction": { + "parts": [ + { + "text": ( + "You help maintain a synthetic Python project. Read the relevant files before suggesting a fix. " + "Explain which file you need next, then call a read-only tool. Never change files. Use the " + "chat_completions client for this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect." + ), + }, + ], + }, + "tools": [ + { + "function_declarations": [ + { + "name": "search_files", + "description": "Search source and test files for a literal term.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + }, + }, + { + "name": "read_file", + "description": "Read a range of lines from a project file.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + }, + }, + ], + }, + ], + "generationConfig": { + "max_output_tokens": 1024, + "thinkingConfig": {"thinkingLevel": "high", "includeThoughts": True}, + }, +} + + +VERTEX_AI_GEMINI_GENERATE_CONTENT_RESPONSE: Final[Mapping[str, JsonValue]] = { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": ( + "**My Thought Process on Analyzing the Request and Planning the Next Action**\n\nRight, so my primary " + "objective here is to fulfill the user's request while adhering strictly to the system's operational " + 'guidelines. The user has specifically asked me to "Read the validation file first so we can ' + 'understand how zero and negative quantities are handled." This is a direct instruction that aligns ' + 'perfectly with the overarching system instruction to "Read the relevant files before suggesting a ' + 'fix."\n\nMy internal checklist, based on the system\'s directives, also reminds me to:\n* "Explain ' + 'which file you need next, then call a read-only tool."\n* "Never change files." (This is a ' + 'foundational rule, of course.)\n* "Use the chat_completions client for this project review." (This ' + "context is already established.)\n* Crucially, and directly relevant to the user's query: " + '"Consider zero, negative and non-integer quantities before choosing the next file to ' + 'inspect."\n\nNow, looking at the files I have access to, I see `shop/validation.py` and ' + "`tests/test_validation.py`. The user's request directly points to `shop/validation.py` as the next " + "file to examine for validation logic. This makes sense, as the name itself implies it's where the " + "core validation rules reside.\n\nBefore I jump straight into invoking a tool, the system requires me " + "to articulate *why* I'm selecting this file and *what* I intend to do with it. So, I need to " + "explain that I'm going to inspect `shop/validation.py` specifically to understand its handling of " + "quantities, particularly concerning zero, negative, and non-integer values, as prompted by the user " + "and reinforced by the system's guidance.\n\nFollowing that explanation, I'll then execute the action " + "by calling a read-only tool, `read_file`. I'll specify `shop/validation.py` as the path, and to get " + "a good initial overview, I'll plan to read a reasonable chunk, say from line 1 to around line 50, " + "to capture the initial validation logic. This seems like a good starting point to gather the " + "necessary context for the user's specific inquiry.\n\n" + ), + "thought": True, + }, + { + "text": ( + "To see how quantities (including zero, negative, and non-integer values) are validated, I need to " + "read `shop/validation.py` next." + ), + }, + { + "functionCall": { + "name": "read_file", + "args": {"start_line": 1, "end_line": 50, "path": "shop/validation.py"}, + "id": "call_267302", + }, + "thoughtSignature": VERTEX_AI_GEMINI_THOUGHT_SIGNATURE_0, + }, + ], + }, + "finishReason": "STOP", + }, + ], + "usageMetadata": { + "promptTokenCount": 246, + "candidatesTokenCount": 65, + "totalTokenCount": 642, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 246}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 65}], + "thoughtsTokenCount": 331, + }, + "createTime": "2026-10-15T01:13:21.457273Z", + "responseId": "sSjQarn0G-Wb4_UPmobfOA", +}