From 0c91bd7fdb1435235bd8e2855b883d41b688709d Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 14 Oct 2026 17:30:36 -0700 Subject: [PATCH] test(translation): share rich request and response templates (#46546) * test(translation): enrich Sonnet 5.5 base conversations * test(translation): capture prompt caching and explicit reasoning * refactor(translation): share endpoint request and response templates --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/translation/AGENTS.md | 9 + tests/integration/translation/README.md | 29 ++ .../chat_completions/bases/anthropic.py | 77 +---- ...at_completions_prompt_caching_anthropic.py | 231 ++++++++++++++ ...st_chat_completions_streaming_anthropic.py | 19 +- .../translation/chat_completions/templates.py | 174 +++++++++++ .../translation/messages/bases/anthropic.py | 138 ++++++--- .../test_messages_prompt_caching_anthropic.py | 84 +++++ .../translation/messages/templates.py | 84 +++++ .../translation/responses/bases/anthropic.py | 133 ++++---- ...test_responses_prompt_caching_anthropic.py | 208 +++++++++++++ .../translation/responses/templates.py | 287 ++++++++++++++++++ .../translation/templates/__init__.py | 0 .../translation/templates/anthropic.py | 150 +++++++++ 14 files changed, 1428 insertions(+), 195 deletions(-) create mode 100644 tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_anthropic.py create mode 100644 tests/integration/translation/chat_completions/templates.py create mode 100644 tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_anthropic.py create mode 100644 tests/integration/translation/messages/templates.py create mode 100644 tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_anthropic.py create mode 100644 tests/integration/translation/responses/templates.py create mode 100644 tests/integration/translation/templates/__init__.py create mode 100644 tests/integration/translation/templates/anthropic.py diff --git a/tests/integration/translation/AGENTS.md b/tests/integration/translation/AGENTS.md index 33d176256e3..aaf18ae21ee 100644 --- a/tests/integration/translation/AGENTS.md +++ b/tests/integration/translation/AGENTS.md @@ -2,6 +2,15 @@ Exact translation cases on the shared fake provider. `README.md` here has the case fields, deployments and capture steps +## Templates + +Reuse the client request and expected response in `/templates.py`, and the upstream endpoint's +request and response in `templates/.py`. Model cases bind those templates with dictionary +unpacking and explicit model-specific overrides. Templates omit `model`. Keep exact captured values, +replace nested values rather than mutating them, and never call production translators to build an +expectation. Native passthrough fields may share literal data. The Sonnet 5.5 pilot demonstrates this +across Chat Completions, Messages and Responses, with cache-hit response variants + ## Naming Each model gets one complete base `TranslationTestCase` in `translation//bases/.py`, diff --git a/tests/integration/translation/README.md b/tests/integration/translation/README.md index 424772d8e43..b6cbc877a9c 100644 --- a/tests/integration/translation/README.md +++ b/tests/integration/translation/README.md @@ -4,6 +4,35 @@ These tests check one request through the proxy as literals on a `TranslationTes Some response fields are generated by LiteLLM on every run, such as a chat completion's `id` and `created` or a response's `id`, `created_at` and output message `id`. A case writes those as `unittest.mock.ANY`, so the field must still be present but any value passes. Use `ANY` only for a value LiteLLM generates, never for one copied from the provider reply +## Shared templates + +Each client endpoint has a request and an expected response in `/templates.py`. Each upstream provider endpoint has a request and a captured response in `templates/.py`. All three client endpoints in the Anthropic pilot share the `/v1/messages` provider templates. Native Messages reuses the provider response shape. These are plain dictionaries, not builders that call production translators + +The base template count is twice the number of client endpoints plus twice the total number of supported upstream endpoint formats. Models select templates and override their differing fields, rather than creating another template per model. Streaming and other distinct shapes need variants + +For example, the Chat Completions Sonnet base binds four templates + +```python +litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "anthropic/claude-sonnet-5-5"}, +expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-sonnet-5-5"}, +mock_provider_response={**ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-sonnet-5-5"}, +expected_litellm_response={**CHAT_COMPLETIONS_RESPONSE, "model": "anthropic/claude-sonnet-5-5"}, +``` + +The shared samples come from the Sonnet 5.5 captures. Adding another model reuses the endpoint request, with small explicit overrides for unsupported features. Make a live capture with that exact request, inspect its translation, then override differing provider and client response fields in the model's case. Do not relabel the Sonnet capture as live evidence for another model. A new provider endpoint needs its own wire templates, not another client conversation + +The templates omit `model`, so the case binds the deployment, provider model and returned model explicitly. Keep captured IDs, content, thinking signatures and usage exact. The Messages and Responses pilot cases retain their endpoint-specific capture differences. Native Messages can share unchanged content and usage between the provider and client response, while translated response fields remain independently recorded expectations + +Build overrides with dictionary unpacking and new nested values, never mutate a template or its nested values. Do not compute expected translations by calling LiteLLM. When editing a shared template, review and run every case that uses it. Older bases keep their existing captures until migrated with matching live evidence + +## Rich capture examples + +Use the Anthropic `CLAUDE_SONNET_5_5_TEST_CASE` bases as the rich conversation examples. They contain synthetic system instructions, multiple turns, two read-only tool definitions, a prior tool call and result, and a follow-up request with `cache_control`. Explicit high reasoning effort uses `reasoning_effort` for Chat Completions, `reasoning.effort` for Responses, and native `thinking` plus `output_config.effort` for Messages. Their provider bodies were captured separately for each client endpoint, including tool calls, thinking signatures and full usage. Keep the endpoint's native request shape and inspect the translation before recording its expected output. A changed LiteLLM output is not automatically correct just because the proxy returned it + +Each rich base captures a real provider cache write. Its derived `CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE` in `prompt_caching/` uses the identical request and the subsequent real cache-hit response. CI replays these bodies independently through the fake provider, checking request translation and usage conversion without requiring a live cache or test ordering. `cache_control` configures the provider's prompt cache, while `cache: {"no-cache": true}` prevents LiteLLM's response cache from bypassing the provider + +Keep specialized feature captures paired with their original requests. If enriching a base changes an inherited request, recapture that feature or override its request explicitly, as the separate Anthropic streaming case does. The maintained provider/model target list lives only in the daily translation-sync automation prompt, not in a second repository manifest + `StreamingTranslationTestCase` stores the stream request, exact provider request and provider and LiteLLM SSE frames. The runner compares parsed frames exactly and checks the final message they join into against `expected_litellm_response`, with streaming cases living in `/streaming/` Deployments used by translation tests are shared by the whole suite and declared in `proxy_config.yaml` under `model_list`, with `model_name` equal to the litellm model string, `api_base: http://127.0.0.1:8191` and a synthetic key. A case names the deployment literally in its client request. The fake provider behind them is the `provider` fixture: one `wire_server` on port 8191 inside the pytest process, started the first time a test asks for it. A test queues its replies with `provider.expect(...)` and reads what the proxy sent with `provider.received()`. Tests that use it run one at a time. After each test the fixture fails if a queued reply was never requested or a received request was never read, and before each test it fails if a request arrived in between, naming the previous test. It refuses to start under pytest-xdist, so the `mcp` and `cost` groups cannot use it. Client requests carry `"cache": {"no-cache": True}` because the proxy caches responses in Redis diff --git a/tests/integration/translation/chat_completions/bases/anthropic.py b/tests/integration/translation/chat_completions/bases/anthropic.py index 208308b17c2..a291a167a64 100644 --- a/tests/integration/translation/chat_completions/bases/anthropic.py +++ b/tests/integration/translation/chat_completions/bases/anthropic.py @@ -2,6 +2,8 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.templates import CHAT_COMPLETIONS_REQUEST, CHAT_COMPLETIONS_RESPONSE +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -332,83 +334,16 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/chat/completions", - litellm_request={ - "model": "anthropic/claude-sonnet-5-5", - "max_tokens": 64, - "messages": [ - {"role": "system", "content": "You are a terse assistant."}, - {"role": "user", "content": "Say hello."}, - ], - "cache": {"no-cache": True}, - }, + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "anthropic/claude-sonnet-5-5"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", "anthropic-version": "2023-06-01", "content-type": "application/json", }, - expected_provider_request={ - "model": "claude-sonnet-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], - }, - mock_provider_response={ - "model": "claude-sonnet-5-5", - "id": "msg_011CfgDzd9tYt5h3TSZTehYZ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, - }, - expected_litellm_response={ - "id": ANY, - "created": ANY, - "model": "anthropic/claude-sonnet-5-5", - "object": "chat.completion", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello.", - "role": "assistant", - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, - }, - } - ], - "usage": { - "completion_tokens": 6, - "prompt_tokens": 23, - "total_tokens": 29, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 6}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 23, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "inference_geo": "global", - "service_tier": "standard", - }, - }, + expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-sonnet-5-5"}, + mock_provider_response={**ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-sonnet-5-5"}, + expected_litellm_response={**CHAT_COMPLETIONS_RESPONSE, "model": "anthropic/claude-sonnet-5-5"}, ) CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( diff --git a/tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_anthropic.py b/tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_anthropic.py new file mode 100644 index 00000000000..13a6ac427bc --- /dev/null +++ b/tests/integration/translation/chat_completions/prompt_caching/test_chat_completions_prompt_caching_anthropic.py @@ -0,0 +1,231 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.runner import assert_translation + +CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0: Final = ( + "CAQSkgkKEAgSGAI4AUIIdGhpbmtpbmcSDF+t4V3r32toxL3AdhoMUGxH3WEiPl1PzrbEIjBXWl/8pilac6QmCoAf2V6ccw1t8goO+vJrZI+Q" + "P8MYvCDZVMnVW27phzm4id3agp0qrwjlUCjy0oY0HLDs74MWH3D6CY+kO2DNnYDc63CzJfbYmw0R/+pCIuRdqi7uZnep3fjdaHVw27Et0zDr" + "mohz3SSc9P00Q0lereo8tOVPsiRezXyu5bMXUhNYM14rmH8GI3eMB8dZEZQ2iLomWTqROOfDDCNAiyfBhbJK+cMq2jrPhr19xwwcut3esQnf" + "5ZhFtfQcHz4GSGWXnQEU53+k7Lixm89JiJrnmls9ZTzFeSkh3FFnX6cnJxzWLgm5jK+a3m1eKjO41QccoioBDI4wgzImAiUTjIptLDB6q9lg" + "/SW5sc3YUxQPNgVJxdYzbRrKSiBgFli6C5n+yXHai4uA/4K0+uPZ2JbUH1Ofsxs+6x0Jcgawenu8GKAXuaOoqv/DWsWQwm8kNOpYaEgKm0Ld" + "nX3q8FljhroX0DwHabcKGjh27rMHpSh5JomVChvFOGeUaiO4jKeaD/bY9CmJMTlc5anJV35LxHzvhYHNiG9vFcQjmOopIIq8pe/F9JKyC4tO" + "v0bCH9jp0dUljq4X55fQZ2xOz2v8VpjH2cVK41N31/ifyDV2fP3BiIP0I9rS5+aos1DzhW22bL5Bsb6qinwk2fRiQXPlFitHITxYY2ORep+E" + "t7edv5h4wN6iYM9YrVeTJr7gcUdcxeFlR2rHWl+8cyVZrH6WroyWU0DZzsaNCNlv78TBLYRO3ZRJz+mm07GP8zqVVfEHYORcyVSw9uc518uE" + "t6Xc9cZnRzIc94675OeRflgOPKHcxDjWS7+C91g4mWx8UONWOxNkSj1k7FpP738A+89XCe0X+18JtMKrbZxwnhKZzsLbFQxARsff9dyhjq5D" + "3ArlPpKF0DquTWuBwf7Wd9BXBNcPxeQbaPIeXFSfhmFPtp5YRQN85yiEaKvOmMxrfMYqEZ/Uu0CP1/j511y9umlnbiEWVqxMHtvqwvdtNiVl" + "q71QUuNiCVZ4+u7HIgv87ZnuzFBzLzvIlzEPn6Kj+RCifesqVqj9VXw4rHU6ofQHKmePRw3D0t9cXSxRscv6Uks0CY31x0h2X2CTxu5jYG0O" + "dTyMjaVyoeCtyfHf6DqrCsWJ1U6AzU+kfxIWXRFR+vNmHDlTOSUdtX/HlUB/dpuHAy1l2l9pyF0AuxAIMujraal8Wni899TnkFXSXSy8PqKM" + "wWjygTVnx2HOvpUveL2xzoepv9kmn90dIax0vNs0fVTDskjls4A1C6xs2NfEH2LspxgcXaMbXBXoz2sm8b8d0NNQZewp/lsFgCbmpb/qjg7c" + "pyXRKnv/sJARoW/wh27QoFQOM179O2OdGv3gSqRA3XFNuUoJlRxxyQZylTd39nnqXUFL79Hg39eNKWJZ6qF75R+D6nWEK3ZNSsjKXf53HsPw" + "SF2xdv6nR4Pa58WgGePkzpeInZuZR8ZAgrkXp8zCMSqnKlZfIihm3vkYAQ==" +) + +CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1: Final = ( + "CAQSgAcKEQgSGAI4AUIJbmFycmF0aW9uEgyAZome6lAFZTDHqEAaDPcIxC714yOPmf/uhCIwELGsZh9B7fdio/ac5XWaTOpTTFoN8Viy24I7" + "n3N4t++qwtl7piCH4qcmDsKRxv0AKpwGVxJEtEWtjrN0np/TwG8KMd8pUAyJBHdVgzT/KEXb3jFIt8Pk7XfQrBfjO/HNUl6l1ETczoLm1Spv" + "6RAREIVw3VxkMZ5niHMly/eFOkfihJRQc1PvnpbcPB+fTOHknrq8T6srlYML2ZurKAb7W0E6bucp4QLdo3Gt4zKzLS2Xsl3jKEElZL8/U0Ra" + "s9ymfh/7dO0mIA7QdRwjlJKggy0CzpfWEXj3x8OQSUwMgyYHd/fb5qFUhc6WSIMQt3YI1xrsex8YaI3r2emx/7DaeABpqCYiZAVRrwMSbL/r" + "6sjBgJ/zGaz+XLIPL3flMvhLeAY7/5rcTQ3ub7OjAxRkmxxL0zkK8IrMLMankXHsniXj+hQA8U+ZUL4ybfXyXb7YZPHr0F/3zf/z4KJ4Q7UU" + "OJ96n/t1B3c+tREN4c9hRbDIuWlX2WsdSdXFkHZxTO2BCeZ0soempDM+CW5QMIVQ/8fPWwyJi1ZcDq3HmL6NtmhYbswzoOweOcSwrtK+3Td1" + "GvqOMq1DE/HhXflLVjl5wlaTj7J56GJOCgq+Aw08pJLPBjIoKxGzHrKEVO60JDDj4DLbxi0LoKVtqMzJXdkChdnFYs5ksYHSIEhmYxqSgY6r" + "ZJ170gU8Kluarhm5GmiB1PPnRCvKEbfGNsZge93iVGMqRsWaScy4Z/pi8a5lwmKATZVVHyan93TXB1lRET9s9DG+8mt2IIm3vrMgDwfVD6pL" + "fBlxTkxhn+ynRLnEKsuiZ3x2WHTAXIPSXPI8lYRXuJjmIPb0vD6XfPbTDLMfvVIJr7UT/+FtOX5ELKPA3HCVnCQniUNYQDeY0ZdtvmfTg19N" + "RSbPH552/fk7wxPgFY/HeETIMHA6lTWfcyM7r7EJaA2nqVjoC+7UP8Xid2+uX8yaT02c18EqvhhwbD6GuoIUIOFXPMct1EdxpGYeqg1BRrAM" + "ciJCkxRTyJ4ySotKH8dwW7r6xk+ZppOoU3suJBLRH+mSqh1JLp2vSoIEezxk4+gviPA79H5I+WfN0D8R8irrnfiUGLpsO1utDIeLE9GqmEEv" + "2GTYRu5xn9KfzxgB" +) + +CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + "id": "msg_011Cg32q7o5hyKPksYEKC7na", + "content": [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file first, though reading the test " + "file too would be independent. I'll stick with their explicit instruction " + "and read the validation file first.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see how `validate_quantity` handles " + "zero, negative, and non-integer values, and simultaneously review the " + "zero-quantity test in `tests/test_validation.py`.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_01J1PTHbwMYmFEqmpJuxuFGL", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_01JSedGM3Q4DBt1C2aix1QdH", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 50}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 444, + "output_tokens_details": {"thinking_tokens": 248}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_01J1PTHbwMYmFEqmpJuxuFGL", + "type": "function", + }, + { + "index": 3, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 50}', + "name": "read_file", + }, + "id": "toolu_01JSedGM3Q4DBt1C2aix1QdH", + "type": "function", + }, + ], + "reasoning_content": "The user asked to read the validation file first, " + "though reading the test file too would be independent. " + "I'll stick with their explicit instruction and read " + "the validation file first.\n" + "\n" + "I'll check `shop/validation.py` to see how " + "`validate_quantity` handles zero, negative, and " + "non-integer values, and simultaneously review the " + "zero-quantity test in `tests/test_validation.py`.\n" + "\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user asked to read the validation file " + "first, though reading the test file too " + "would be independent. I'll stick with " + "their explicit instruction and read the " + "validation file first.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see how " + "`validate_quantity` handles zero, " + "negative, and non-integer values, and " + "simultaneously review the zero-quantity " + "test in `tests/test_validation.py`.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1, + }, + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user " + "asked to read " + "the " + "validation " + "file first, " + "though " + "reading the " + "test file too " + "would be " + "independent. " + "I'll stick " + "with their " + "explicit " + "instruction " + "and read the " + "validation " + "file first.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check " + "`shop/validation.py` " + "to see how " + "`validate_quantity` " + "handles zero, " + "negative, and " + "non-integer " + "values, and " + "simultaneously " + "review the " + "zero-quantity " + "test in " + "`tests/test_validation.py`.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1, + }, + ], + }, + "content": None, + }, + } + ], + "usage": { + "completion_tokens": 444, + "prompt_tokens": 880, + "total_tokens": 1324, + "completion_tokens_details": {"reasoning_tokens": 248, "text_tokens": 196}, + "prompt_tokens_details": { + "cached_tokens": 876, + "text_tokens": 4, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 876, + "inference_geo": "global", + "service_tier": "standard", + }, + }, +) + + +@pytest.mark.parametrize( + "case", [CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE], ids=[CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE.id] +) +def test_chat_completions_prompt_cache_hit_anthropic( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py index ca852a92f94..ccba51cff23 100644 --- a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py +++ b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py @@ -11,10 +11,25 @@ from integration.translation.runner import assert_streaming_translation CLAUDE_SONNET_5_5_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( scenario="streaming", litellm_endpoint=CLAUDE_SONNET_5_5_TEST_CASE.litellm_endpoint, - litellm_request={**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, "stream": True}, + litellm_request={ + "model": "anthropic/claude-sonnet-5-5", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + "stream": True, + }, expected_provider_endpoint=CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_endpoint, expected_provider_headers=CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_headers, - expected_provider_request={**CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, "stream": True}, + expected_provider_request={ + "model": "claude-sonnet-5-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], + "max_tokens": 64, + "system": [{"type": "text", "text": "You are a terse assistant."}], + "stream": True, + }, mock_provider_frames=( SSEFrame( event="message_start", diff --git a/tests/integration/translation/chat_completions/templates.py b/tests/integration/translation/chat_completions/templates.py new file mode 100644 index 00000000000..67f4dd8a2e8 --- /dev/null +++ b/tests/integration/translation/chat_completions/templates.py @@ -0,0 +1,174 @@ +from collections.abc import Mapping +from typing import Final +from unittest.mock import ANY + +from integration.translation.templates.anthropic import CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0 +from pydantic import JsonValue + +CHAT_COMPLETIONS_REQUEST: Final[Mapping[str, JsonValue]] = { + "cache": {"no-cache": True}, + "max_tokens": 1024, + "messages": [ + { + "role": "system", + "content": "You help maintain a synthetic Python project. Read the relevant files " + "before suggesting a fix. Explain which file you need next, then call a " + "read-only tool. Never change files. Use the chat_completions client for " + "this project review. Consider zero, negative and non-integer quantities " + "before choosing the next file to inspect.", + }, + { + "role": "user", + "content": "The checkout rejects quantities of zero. Find the validation code and its tests.", + }, + { + "role": "assistant", + "content": "I will locate the quantity validation and its tests.", + "tool_calls": [ + { + "id": "toolu_synthetic_search_01", + "type": "function", + "function": { + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "toolu_synthetic_search_01", + "content": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", ' + '"line": 7, "text": "def test_zero_quantity():"}]}', + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Read the validation file first so we can understand how zero and " + "negative quantities are handled.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + ], + }, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search source and test files for a literal term.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a range of lines from a project file.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + }, + ], + "reasoning_effort": "high", +} + +CHAT_COMPLETIONS_RESPONSE: Final[Mapping[str, JsonValue]] = { + "id": ANY, + "created": ANY, + "object": "chat.completion", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I need `shop/validation.py` first. It holds `validate_quantity`, " + "which decides how zero, negative and non-integer values are " + "treated. I'll read the area around line 18.", + "role": "assistant", + "tool_calls": [ + { + "index": 2, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "name": "read_file", + }, + "id": "toolu_01NdDgeoKnYQe6YXxzc3XDmF", + "type": "function", + } + ], + "reasoning_content": "I'll follow the user's request and read the validation " + "file first, focusing on a reasonable range around line " + "18.\n" + "\n", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll follow the user's request and read " + "the validation file first, focusing on a " + "reasonable range around line 18.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + } + ], + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "I'll follow " + "the user's " + "request and " + "read the " + "validation " + "file first, " + "focusing on a " + "reasonable " + "range around " + "line 18.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + } + ], + }, + }, + } + ], + "usage": { + "completion_tokens": 235, + "prompt_tokens": 880, + "total_tokens": 1115, + "completion_tokens_details": {"reasoning_tokens": 80, "text_tokens": 155}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 4, + "cache_write_tokens": 876, + "cache_creation_tokens": 876, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "service_tier": "standard", + }, +} diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py index fbcec006d71..1a3a054ab4e 100644 --- a/tests/integration/translation/messages/bases/anthropic.py +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -1,6 +1,10 @@ +from collections.abc import Mapping from typing import Final from integration.translation.case import TranslationTestCase +from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGES_RESPONSE +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE +from pydantic import JsonValue CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -343,16 +347,87 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, ) +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSjggKEAgSGAI4AUIIdGhpbmtpbmcSDH0nWsNq6ztJU4FmSRoMNF3KUUnzhPIVkxnLIjBUD6huzC4a6KowV1auL/9ovi4lnt+8VcIkUJat" + "qASbHEWK6cwcKYp/QaSkeXlRJd8qqwdMqLSP3zIGOAvSrDbirhKmZzy9H6y84DP1WIscCMCeYeRbSdSSU8vpA+d3ZgTOMzXbS3p//HayGi8l" + "xOtDgHN8vgu7I47CXNYhItB3+/k51IwVPFP1Xdf2RuZPhNsYd8/PctETsmY8FZzVih7ZUBT7LYOL6DTbDIM4emZSQl4lMDJtEJ6V+wCsy69f" + "qHCY7PYZQyoNFM3RCghSm+xPU5RP7o4uHq3S0ajPWhw83q2waWANCTKrWuW+VD5ivZq7a/2WzzhsSiJ7oerdbzGO9mNVyl6I9BLoxtSjW3Jm" + "ygUQO99/OAk1rQG7FbzwHO5PsEIT+Ojr1hVmPY9qjsC+6LV5Dm+3vGd4Yg0jygVoU62FjepPeIbfuJAYINo14bFF8QxNNUrEFTtRmWltrD8I" + "WoeZ3bYjzyHdni5l4rFp4Pz2H7IIEoesZrLfoGDVydrrPaBjJx45RIpj/BPuK0GClteaSutn+4DsKcpWLSDA7UGWRFabB+KIKoZa/o8PHFW5" + "pW5KIFShOzXbnDNQcVJrNDZ54pzBkRMCEuGvudJkOuK/A1qqQlE/lYNww2PkW5OBt+WdNMhXqk7R03k54ruEm/W9zEElIPaKBiJ3YQ0TiBrn" + "W8G3CMxusdl9fR45Ku3KIZwTwnlQ0yx+YBNKT1F2rpLHd3sE5k8wf7o6ciZykEmINC4jh+Ygy5Juau1TsVJg0hsCx75IVa0l2oJKanWdIY8v" + "1eoqX4OsVxB1sdlfIA7eqQH0cGca3a5oHcrmarLFDeqM1ne5MbGYJYQm6Ry7zOzUO/SRv/aWGH2qqcXtqfdHvA2C6NFpPqze+yYTU1G913jn" + "nigXprL6tMBOKfNMz12Y3iaFG/BGslJPh5UOfYUqrp0xWRqxm6CazOfW7M75qbzyQIkpKoqGSflJO+TKE0J+52AehMDYEbrSmIYcQZ8ZcCZs" + "pYwk7hwwQzNiLgjQZsheaJYTH05ZV1PRYW8+n4yv87yubjGvRk3QidSzOpYr/reUDOFSZOqIfBVOpwIqbju/ZvyHxaKx2PIBGyOBBu7xSz8u" + "CD3ViTbnYmYs9CMyQ46BdKyb9/cYZ3N4hJ0ucfDir0Q80BQUU52+cFYb1LUPzkz3qL/Cc3fDr3qxjyHCvhm1a1ukpzlakXmuY5edAz0zP6vw" + "qm35U4PeIKQq2j9PUXwJX8SYkbI9c4VmO1z1SeKJHiLzJbOQVvSN1+mhO6aTwjmB2dzTFxW0QlHlWqrwA4vcYWyYK1pHS+4YAQ==" +) + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQShwcKEQgSGAI4AUIJbmFycmF0aW9uEgzgvB/8tNUGq1nfsxwaDG+cT0qM1p11gJfjDSIwtRicc6aAt2GBKWg1GHCUadncKjMPzXWPvDlt" + "+F3ibcjosDbl5nyvWowUp+Vq+AyHKqMG3SAB7dNiY7nLrdCzSD2Xv2LLv0b8hPdf/vnIY73zBhH7rHOjRCRPMqNVGuXfpZVvTCvgyuXbLaAq" + "xHg3O1GexFc8OPnXxBt0jjYXu4N/5yKAQcl+7xwwS18UnSh076vLr47RER5xRmQ5IiPNi43UO8jurj50AEucd6V6C+l85jZNUIsQPEgqKIt/" + "vi9VSDjGkkTbLCdC/DW95wE8SWXkJO1Bs7VjHm8LHNTbYyLzc64FJkBQ+DqFjX0lZCkKLaeJML7nw3W/vo0p/E3XNDVjGgyErJYo243rIsNB" + "rfH0PQacca0w+ipJWRPJng64+Fnyi8KlYV5qz35p4MmvSSIbkoHUZ1IDrP8CSoc3TIvcwCaTPY9b0TkmUuCTBNmvLS1I/x+Y7f9PH+rMO1Ad" + "8PFXePqjR1Hq1smJgZMpjVMGBqKZkBoKNK3g2ScNrjqzkOMo9jYZhpfnDnl4eF02ouuOktuNTuJlPc7L1xFADckbPAkqTyVrUmGJTGL8bOz3" + "cPltf1MzeHMoe9IOG5gxCYHlo3wb53UMOxZJ9dAK4QwdG7LyksN3sSfQ1nqpIVgWZ4iN6wLZt4H21P/MGUddVH6nCRmJSjELG9VqzEXyYquj" + "MgUFtFhS8qYZDE/077wGT0Jw5IYyXTTEHjqV9I+UIb0zaOTWWgBefmnnahJYpq5Wjmrki5nRvAKdPY28gTidvvdbrt5eoJ5h+7kXAYXM8Hh0" + "3LrfnIBfjuBy4/SgHAtLbrWK0jEVXF8QeGL/OP7q8Xdr6Mbb9bxL2JndZi97zn2x/ae1vicszJXsl/QuGNGrShjt6dyPT4nwoClVJX9VIGjv" + "t88CzZ+/s7Ar3jbgOEqVEfPDjtdnlTswfRAhxzTlvWd6axok7j5G0c4bl2TA1EPQiIL6GDpP51o0biQ3NMi5InEZxUnU/Y7e6tqn5GKQLqEn" + "2h9IgVn+rDbcq4AN4FgwrF88sQdxBTCK/9AP76jvcX/WogcIclnu+RzmzHijGcMXxrlqDvIdauQsdUKakPnW2VCmfxBu/bChJpS3CM493jrF" + "PHy3VOS8NxDVLe4HAPDiOM0YAQ==" +) + +CLAUDE_SONNET_5_5_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg32qT4xpsR9qgkrqjiGw", + "content": [ + { + "type": "thinking", + "thinking": "The user asked me to read the validation file first, but since reading the " + "test file is independent, I can fetch both in parallel without violating " + "the intent of their request.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `validate_quantity` in `shop/validation.py` to see how it " + "handles zero, negative, and non-integer values, and also review the " + "zero-quantity test in `tests/test_validation.py`.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_01WcurCDsB8E8W42GKceb8B4", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_01NdHxDmUwwCWpWyaLT2ThJH", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 416, + "output_tokens_details": {"thinking_tokens": 220}, + "service_tier": "standard", + "inference_geo": "global", + }, +} + CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", - litellm_request={ - "model": "anthropic/claude-sonnet-5-5", - "max_tokens": 64, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], - "cache": {"no-cache": True}, - }, + litellm_request={**MESSAGES_REQUEST, "model": "anthropic/claude-sonnet-5-5"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", @@ -360,54 +435,21 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, "model": "claude-sonnet-5-5", - "max_tokens": 64, + "messages": MESSAGES_REQUEST["messages"], + "system": MESSAGES_REQUEST["system"], + "tools": MESSAGES_REQUEST["tools"], "stream": False, - "system": "You are a terse assistant.", - "messages": [{"role": "user", "content": "Say hello."}], }, mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + **CLAUDE_SONNET_5_5_RESPONSE_FIELDS, "model": "claude-sonnet-5-5", - "id": "msg_011CfgDzY6H6Rnp7FC7LsZvm", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, }, expected_litellm_response={ + **MESSAGES_RESPONSE, + **CLAUDE_SONNET_5_5_RESPONSE_FIELDS, "model": "anthropic/claude-sonnet-5-5", - "id": "msg_011CfgDzY6H6Rnp7FC7LsZvm", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, - "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", - }, - "diagnostics": None, }, ) diff --git a/tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_anthropic.py b/tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_anthropic.py new file mode 100644 index 00000000000..fed83a7b19c --- /dev/null +++ b/tests/integration/translation/messages/prompt_caching/test_messages_prompt_caching_anthropic.py @@ -0,0 +1,84 @@ +from collections.abc import Mapping +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.runner import assert_translation +from pydantic import JsonValue + +CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0: Final = ( + "CAQSiQcKEAgSGAI4AUIIdGhpbmtpbmcSDJmy4nsZ6G9GGfujbxoM2xU6lMsHnAjRbgKPIjDCdWjR2/DWUdgGbUsetewZ+xbrYi08H7fVA3Md" + "HGr39Zv/cIwTC3Joe9594mXMJ8MqpgbWIL0Gg3BkRvPNxSzamBJMdkjuwxZbY9RPkkXIDDJ3lVI7s/Rw2jZCsw+sFVRvzehlo9ifeweRgUtJ" + "UQ7Jw5YAXvydQ6afPTlCb5vJuumrJVQtfa2HFER7FOvVmy188kWO/D14rX5YArunnOwN4Tr6WBp36YpACfopAW1H8ysC7qmxlp/C/gXhez+4" + "dCtWb7wU89gE4VKp3VFLTihst3ZbXvqfyP9FFcocJGswk4P2SXO53DoxTQgrkt6apmWOXffk+Bsc/8POf0D0cfx9WspHgm26HhvbkB+aiN8A" + "5ow0jT0QKvAOi/C0epWp/uuTII0GiMHy4pH4VQq37n2Abp1yWfHu5c1OvVtpOn7ySN3rX/xRH3+l4KCIVX06RrYaTWC3n9FzUBI/zr5hMfVC" + "cPxDOFy3rU2Duz/S2yzbhjQIUqTnMEmB93vIfrSWmKXcK37/XTNvHC6HLNpTrIGVmoXvLMJ+k9JcAtvlgtPCEJHULVnruLpMODGuEhKU/CVi" + "kLkK4dzUbhvGIfhb0JrInjP+rxERma3+1arv+FWw8OVdKskcpZB4w6oY0tyYo38sotyvp9G+ZZp+C0rGLoxnC8ZGF1OXQ3MPexY+krWw1TxT" + "WRGtWSSC5xxqzBc5CLBy0D0dV2XLFPpMqqo5m6HkBErnSXLbrERDcMnUZv7/+WpvsIqNNwXgtwmnV5FpO95D0yRfs/ru2KOxxYI6RllfGrX4" + "Rcfd+6umtvFTGIl0YkCoyXAMRplnGSXo5n0xQyXKm1ODptIFml5q/gr31UsdaTvxmbb3hBES+VWskDM+XXNcgS7nNBfyuTT9hnM7SHr1NOqY" + "osZn2ZzSRAZqz8Nz1ZTC7UQLosjWgG90AxwnW4yXhjm2t8BKv82xKeqQwOeKrxBTzPi5+uPfLb3WFBfk2yHBAYz9obGkN/XPISGiTUsX5h9f" + "M3MJOPuFjlToForJFiC14XK5bTJnE34jcVC86adk8rRmoYGVVNx1exFhj0Ewadtfo3z/G5Ufl2d1PC0xfzrtKMzJocR0X+ftm0lI39OLeSPo" + "niP1PuAsUkGk467NXLgH0ulqiRgB" +) + +CLAUDE_SONNET_5_5_CACHE_HIT_RESPONSE_FIELDS: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg32qgbJRU3vZX2TB6vPx", + "content": [ + { + "type": "thinking", + "thinking": "I'll start by reading shop/validation.py since the user asked for that " + "first, then move on to check the related test file after.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I need `shop/validation.py` first, because `validate_quantity` at line 18 holds " + "the rule. I'll read the function and some of the code around it, in case there " + "are helpers or constants it depends on.", + }, + { + "type": "tool_use", + "id": "toolu_01U6Ch75cPbVpbpsBEyPP1AN", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 873, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 254, + "output_tokens_details": {"thinking_tokens": 92}, + "service_tier": "standard", + "inference_geo": "global", + }, +} + +CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + **CLAUDE_SONNET_5_5_CACHE_HIT_RESPONSE_FIELDS, + }, + expected_litellm_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, + **CLAUDE_SONNET_5_5_CACHE_HIT_RESPONSE_FIELDS, + }, +) + + +@pytest.mark.parametrize( + "case", [CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE], ids=[CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE.id] +) +def test_messages_prompt_cache_hit_anthropic( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/templates.py b/tests/integration/translation/messages/templates.py new file mode 100644 index 00000000000..9e8e4a0b053 --- /dev/null +++ b/tests/integration/translation/messages/templates.py @@ -0,0 +1,84 @@ +from collections.abc import Mapping +from typing import Final + +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_RESPONSE +from pydantic import JsonValue + +MESSAGES_REQUEST: Final[Mapping[str, JsonValue]] = { + "cache": {"no-cache": True}, + "max_tokens": 1024, + "system": "You help maintain a synthetic Python project. Read the relevant files before suggesting a " + "fix. Explain which file you need next, then call a read-only tool. Never change files. Use " + "the messages client for this project review. Consider zero, negative and non-integer " + "quantities before choosing the next file to inspect.", + "messages": [ + { + "role": "user", + "content": "The checkout rejects quantities of zero. Find the validation code and its tests.", + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "I will locate the quantity validation and its tests."}, + { + "type": "tool_use", + "id": "toolu_synthetic_search_01", + "name": "search_files", + "input": {"query": "quantity", "paths": ["shop/", "tests/"]}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_synthetic_search_01", + "content": '{"matches": [{"path": "shop/validation.py", "line": 18, ' + '"text": "def validate_quantity(quantity):"}, {"path": ' + '"tests/test_validation.py", "line": 7, "text": "def ' + 'test_zero_quantity():"}]}', + }, + { + "type": "text", + "text": "Read the validation file first so we can understand how zero and " + "negative quantities are handled.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + }, + ], + }, + ], + "tools": [ + { + "name": "search_files", + "description": "Search source and test files for a literal term.", + "input_schema": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + { + "name": "read_file", + "description": "Read a range of lines from a project file.", + "input_schema": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + ], + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "high"}, +} + +MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = ANTHROPIC_MESSAGES_RESPONSE diff --git a/tests/integration/translation/responses/bases/anthropic.py b/tests/integration/translation/responses/bases/anthropic.py index e084ea60793..098a62cb4b0 100644 --- a/tests/integration/translation/responses/bases/anthropic.py +++ b/tests/integration/translation/responses/bases/anthropic.py @@ -2,6 +2,13 @@ from typing import Final from unittest.mock import ANY from integration.translation.case import TranslationTestCase +from integration.translation.responses.templates import ( + CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + RESPONSES_REQUEST, + RESPONSES_RESPONSE, +) +from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", @@ -388,13 +395,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", - litellm_request={ - "model": "anthropic/claude-sonnet-5-5", - "max_output_tokens": 64, - "instructions": "You are a terse assistant.", - "input": "Say hello.", - "cache": {"no-cache": True}, - }, + litellm_request={**RESPONSES_REQUEST, "model": "anthropic/claude-sonnet-5-5"}, expected_provider_endpoint="/v1/messages", expected_provider_headers={ "x-api-key": "synthetic-anthropic-key", @@ -402,83 +403,67 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={ + **ANTHROPIC_MESSAGES_REQUEST, "model": "claude-sonnet-5-5", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Say hello."}]}], - "max_tokens": 64, - "system": [{"type": "text", "text": "You are a terse assistant."}], + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider " + "zero, negative and non-integer quantities before choosing the next file to " + "inspect.", + } + ], }, mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-sonnet-5-5", - "id": "msg_011CfgDzd9tYt5h3TSZTehYZ", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], - "container": None, - "stop_reason": "end_turn", - "stop_sequence": None, - "stop_details": None, + "id": "msg_011Cg32qrmyWNAPnHJhxBL3M", + "content": [ + { + "type": "thinking", + "thinking": "The user wants the validation file read first, but since the test reads are " + "independent, I can just include both in the same batch since it's " + "effectively still prioritizing validation.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see how `validate_quantity` handles " + "zero, negative, and non-integer values, and read `tests/test_validation.py` " + "alongside it to confirm the expected behavior.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_01GYseiK8hh9QxfWJ6HbiAt3", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_01U3TuUTifkSBLVzyTFnbMN1", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], "usage": { - "input_tokens": 23, - "cache_creation_input_tokens": 0, + "input_tokens": 4, + "cache_creation_input_tokens": 873, "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 6, - "output_tokens_details": {"thinking_tokens": 0}, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 509, + "output_tokens_details": {"thinking_tokens": 313}, "service_tier": "standard", "inference_geo": "global", }, - "diagnostics": None, - }, - expected_litellm_response={ - "id": ANY, - "created_at": ANY, - "error": None, - "incomplete_details": None, - "instructions": "You are a terse assistant.", - "metadata": {}, - "model": "anthropic/claude-sonnet-5-5", - "object": "response", - "output": [ - { - "type": "message", - "id": ANY, - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello.", "annotations": []}], - "phase": None, - } - ], - "parallel_tool_calls": False, - "temperature": None, - "tool_choice": "auto", - "tools": [], - "top_p": None, - "max_output_tokens": 64, - "previous_response_id": None, - "reasoning": None, - "status": "completed", - "text": {}, - "truncation": None, - "usage": { - "input_tokens": 23, - "input_tokens_details": { - "audio_tokens": None, - "cached_tokens": 0, - "cached_tokens_details": None, - "image_tokens": None, - "text_tokens": 23, - "video_tokens": None, - "cache_write_tokens": 0, - }, - "output_tokens": 6, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 6}, - "total_tokens": 29, - "cost": None, - }, - "user": None, - "store": None, - "provider_specific_fields": {"citations": None, "thinking_blocks": None}, }, + expected_litellm_response={**RESPONSES_RESPONSE, "model": "anthropic/claude-sonnet-5-5"}, ) CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( diff --git a/tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_anthropic.py b/tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_anthropic.py new file mode 100644 index 00000000000..d185eaca4de --- /dev/null +++ b/tests/integration/translation/responses/prompt_caching/test_responses_prompt_caching_anthropic.py @@ -0,0 +1,208 @@ +from dataclasses import replace +import json +from typing import Final +from unittest.mock import ANY + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.responses.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.runner import assert_translation + +CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0: Final = ( + "CAQSowkKEAgSGAI4AUIIdGhpbmtpbmcSDOwQCAqgczGjZGBQDhoMT4bVPdH6VMFI48MwIjC029tj1Uerfhj93na2igkejc43fgJVSxspenJp" + "9C9WIR+Pw8eVIU2u2oLaCRh9ZaAqwAgXjHbSSi1epawZB+b2WVRclEY69ZZaFJOtC+iA5VBnzI70bhzfYwUL2Bh9vuETb9UyiGNuXlEjTY73" + "AUNWyLn1JGS1y4ZqwoN5u8TZV9zR7nnPoXTIdAlp8vWz0uyE2UgAlqyNX8XNEq2+CsK9R2RFskyXhd8FRSQ49QPj8e+Dcv0OLLpQrrPIa7Nv" + "aG7A0E/5D/VEU94iC538mQ/Eh2NcwBS+gqCCGbeLhHMvmSZqDUp4s4kGMhBPILcAEx1f1FE1ZCsUSKTPMR91XJsarHkdOjgpgBx2yiNr7EAa" + "qP8DEDTMIUZaM4rHuJfCIE2Gu/KOvBaCV5b8gl4HhvONjxfnRqoDkwB79hhB5kARXXcoYPigyJAebXTUKogtdfrLlqJ18KqFiq+gwLk2IP5v" + "l78q3TNiDz4hWoMFdXeaffH2++6FscuZU3im68D2aSWzfUvJoG/5kqFvMKsMUMQ7JTRq8bwdy0Pzc38peH9hgy9OwYjShmmp/lX+8tH4Mpu9" + "4ScwOx0oCKdlAJyFBd4I9kOJ03SM2bZI/NkWTZ11LrtLNqxUpdzESIuaPqcwpBlMQu09oKUthvL7rs5U4WhOD7V+l92UFphiR+hI2Wc+d7Ys" + "kh5PgGpiIGxq5QpMGUOwgy/+/KyZcHc3EDAD4oY/BVpqc4zjjOdFOc6KvtKLn1lqZc47+XL2HxI0Kh7qPqCue3yArxkn16Jk+bdinaJPfVQI" + "rKxIav0gWowtmxcjFOVYXRDzhlaqOfnGZOLDOwJ8TpESRBxKWJ3fYrUjLZjY6X9MdPWQQ88h/DsQijwqXNQFYs/hMOMqEYcW7oIMmbQXv/KT" + "05nX5aMMo0AFD3Iych6bVwNC0b0S87wSolWLBBhnhYyEUJfAW6p28ARtLdkJlqrsOSupzMkKxTxjXafEbM0GcObF9FgaeDh/3gpG9ahFex8W" + "MaBqIakbogrEeQrrt21DPgeb6lWsEMj6lEupfmY5a3Ch2fsKYIFmvvQ4JN9b5C0K7H3GFfH4qTclqTEegX4Gzm7tjuwQsv9INVilYVUkpMgo" + "7JD52wVVtqSJqGgQ8xKLWXR1M5oMjGCE2bRhbgbVfMarvhJW17FJvS/7Yugcgxu/n/dhmo6pxFHlSujiPOBODr8fWPGJKwhpGqKdi+BUQyGF" + "b41hPjRZEuyYSO7OdmqblZ4MlkHFLt7CfCvQQZePBvNyDniiCuwZBRgO3zlRO5VKVsom3KkzZfG+tsfzKwMMrNeN1sQ7udJsp2giBRN1wBu6" + "LBYfs8ASv8Y1OEmFVew5/aByf7oGaH74DlxLi/E0cquY37LiUWl1iTx4SS13wv0NpMaU03X6ahmsEpdHbR2E2I8sBkn8F7b5ydnWpp9OT8Tg" + "9tWMVfprteYjcVkJoKjkp4M7QG4jKSb3/2eofN44aPeX3KfkpbjkMONZkQ7A8z4o1ezDw2swOGVGUBgB" +) + +CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1: Final = ( + "CAQSmgcKEQgSGAI4AUIJbmFycmF0aW9uEgytCDiV0k0xcr3T9oYaDBoLSK23c8n6+DLIhCIwebnBjlb04xHa2geO2nvLWXbdL/mEVCXz+MjE" + "oDSqFLe0Ivh+hn+mIUtg83a3B7NDKrYGXA1HllbZ1yM4570ouFk4bKliGKHWdi0coYw1u9YGlThcas5S7s1hL2+R1hqj0QBP/M/hDMFNbf/1" + "T/idB+7QKbXFsYtonKoTlaQdwhXJehGZfIPNXFM7TYE8hB1KFnJpkJGLM2YU5j42Y1HL0kTh8lNvTdsl2GfF4elrtPAJjeAHEME2b4WfGEbp" + "gJbPFJ6qPWOaVPJbYa7ZpdlOcGYviY9uiXDrXdUNkJx+fQgZ78pX5MIZPUeCkiGIDmxQJtuJC/961w9DEutBOy2KuaQHkuWDZqRCeFAzgW9s" + "hmvT+jo5SxPDtn5E7KTt8DoDoClb6VNEGd+5BX9SSIfl39XyWKsrT7fs/dXDzicl4wUYbD527ZLWM6kJd3PpWcV6He3ltvELyXjYjCGv79+D" + "pM87AwmmpkuYiuTyXrwdDBRfHSf00eoLyFzW/FZNYanwcbTTrReiW1ehhGivaLe1v0ppO2NLSHli2u1+xpdR8bK8Nn8J4q+QnCwON70fuqf+" + "3sQLrspyOO7PlrpV2Ybq8Sng0dC0mBAAv7CpHeR1IaQPC936cfpL7bbB75/m15Pqy2R8A257/g3zZIA53oxwW2B3y/UNzL5m6XSzpmLaYmRM" + "z4ahp50+rytQTCqjG0mr27g2vWerU8U+qqjyzTKrT795/CkVDQ2koodup6jLsfwx1W6bq5aJRVYUEY1ZJKywZylilmWrfTz8MZ4TajBx/yUu" + "5X12jCdjzTOLVdGeOgD5a2lsxZzgU/Y9RgyzeT6RHKNnbjVUXaJa0g5K0MclCfVWFPhuIAwn2Gya/o6eF1k4Pebzr8aIOU8nY3YxXwN9hytO" + "YCwxo4TF2eV8F9ttuBTezOFpJiG2VpvHIRh2/5AVVPxe1c2xWvZvciHHrH14G9rVxmoUzDraCoPiz2XUfCY7e58BRD+wOH3YFnf8uckCmHCj" + "KFYclanFwFJyEfjwssCxcTZvC/AN2aATw4f5WQ1vow9uPO9wxkpc7af/VNQ2MPGB87BaVg/hTRhSgwJLSxZIOKUKMMJzJ/4rgAZRSIJawg33" + "ONNvhW41/bXB33KXdqbvhVIZGkK1YGv8gjMlqIdTfdYDonWeGAE=" +) + +CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_TEST_CASE, + scenario="prompt_cache_hit", + mock_provider_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + "id": "msg_011Cg32rEsFymVyWbaNmV7tB", + "content": [ + { + "type": "thinking", + "thinking": "Since the files are independent, I can just read both in the same call, " + "listing validation first to honor the user's stated order.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see how `validate_quantity` handles " + "zero, negative, and non-integer values, and also read " + "`tests/test_validation.py` to confirm the expected behavior for zero.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1, + }, + { + "type": "tool_use", + "id": "toolu_01RoPyymXeGn7TS6uLnKMkzo", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_01Cv9tx97CJhxergPDGZLTxC", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, + ], + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 873, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 447, + "output_tokens_details": {"thinking_tokens": 251}, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + expected_litellm_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Since the files are independent, I can just read both in the same " + "call, listing validation first to honor the user's stated order.\n" + "\n" + "I'll check `shop/validation.py` to see how `validate_quantity` " + "handles zero, negative, and non-integer values, and also read " + "`tests/test_validation.py` to confirm the expected behavior for " + "zero.\n" + "\n", + "annotations": [], + } + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "Since the files are independent, I can just read both in the same call, listing " + "validation first to honor the user's stated order.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see how `validate_quantity` handles zero, negative, " + "and non-integer values, and also read `tests/test_validation.py` to confirm the " + "expected behavior for zero.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_01RoPyymXeGn7TS6uLnKMkzo", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01RoPyymXeGn7TS6uLnKMkzo", + "namespace": None, + "status": "completed", + }, + { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_01Cv9tx97CJhxergPDGZLTxC", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01Cv9tx97CJhxergPDGZLTxC", + "namespace": None, + "status": "completed", + }, + ], + "usage": { + "input_tokens": 877, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 873, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 447, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 251, "text_tokens": 196}, + "total_tokens": 1324, + "cost": None, + }, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "Since the files are independent, I can " + "just read both in the same call, " + "listing validation first to honor the " + "user's stated order.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see " + "how `validate_quantity` handles zero, " + "negative, and non-integer values, and " + "also read `tests/test_validation.py` " + "to confirm the expected behavior for " + "zero.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_CACHE_HIT_THINKING_SIGNATURE_1, + }, + ], + }, + }, +) + + +@pytest.mark.parametrize( + "case", [CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE], ids=[CLAUDE_SONNET_5_5_CACHE_HIT_TEST_CASE.id] +) +def test_responses_prompt_cache_hit_anthropic( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/responses/templates.py b/tests/integration/translation/responses/templates.py new file mode 100644 index 00000000000..70ae996e785 --- /dev/null +++ b/tests/integration/translation/responses/templates.py @@ -0,0 +1,287 @@ +import json +from collections.abc import Mapping +from typing import Final +from unittest.mock import ANY + +from pydantic import JsonValue + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQSxQoKEAgSGAI4AUIIdGhpbmtpbmcSDJdM5+jcpq2kA3N39xoMeIIhaZ+bur3WodGvIjBSU6GZokT+noSpw7KkJ/YVl9U2kcKaRVBk89gg" + "A9sOJ3s5dwOaPRzfGM9eBlPjMloq4gm8QOkNSzqF4i54b4JhCzRhnQcadRp9p+fVIKrJA2rwqWdSlKKK2kkAkzugndx0wBEhdkxrsbkTDTS2" + "JbXw8xCTdWwP5rJ7Gwd2jVY2GmI60plZW+WcJbLJx4nL/B2rpNWb6xdriK8kKSIovInuhya0hdr8K1/l4+5M72LnFO/S0zFRcBjL3EF7kApi" + "Hr7BwXSsSUJJn6BFsqyN9AkDgiWrdvFU6UzsrkIEEPy+/Mi0a5utQyP5PDGqZfHt3cOeULtfR71DNdt6H7/56g1hRWVig0rrhzRM7N0a3ebP" + "uWtEAbPynqr9OL2IuEwYHFD+UIgUdSyMi8nGwP9vBsMu5UpJ8/79XD3CTth9tzLFRngwCPynkDPsCLqy+iiSRMBFEH5qEZja4PI1Xa1swLzb" + "QRwEFbURPUcGt2P3KSJxMFqpithxIFUlKNawx5DGLOjt6IH4nB4fpSrR/sjynF5RsjD2fWmioB10f2qqWlgqO0nuzYxxO7ES0HFIcutLd9X7" + "u8cQMHZ+FSz3M/IeZhwQloLg4SYi3adMgzLKX/81Re/ZmSgQCrNFPppSAWERaXlzcnreD9FoHHiHVPyRgq1kZffq2YOq7bJfNWfjIBpQsW6S" + "2lmFwH3a2iiA3Jphq5Xvkshpsek+A+6+MrgfPOONA+Hz8WygBDZIEteGRYFTtwNJWTAUlo8fwZnbNWFXq+TyNotKVz1fYrRQSjtR6pKulO9I" + "yzeG3d2UQalbgGpaZQsl2LumVFdXLz9r6DhVUnyzp9cTcMDahLjf0NSAewmp0WNU3vo4j6c7YXsV/Ddje9rXwkKXO8/cmxrDQRJowyNfRtkM" + "XiEMRHLikS+Wdd+T0UbEpBQMF0qjn8aWMfV3adRt3TrMU6xsaKAP0nKXvzz3gXt4UwRTqVuOA25a/dQBMvuvtifuQI0iyC+rl7tSV6vh2Qrj" + "96N7KyNdniQUNKKN/s8QXuPZ+nsxFDnC7hy5vZguZnc97LOFXVltUmUyIa7rHZCFu3Dx16m6t2BdCZZCaL3Z2VglRrYAsUj+0klVoYfJAzsr" + "Fz9VR1y1froAdvB5K+waOAdcSyIG4NxArvXqlNHChqLgSMStGj3eLrtiddboZwNl7TFLd5BJrFBLiEwXjckhsi4nYqLoyrn/F5i3UwqzHV+O" + "xJpKIoLjI+FT0ZappHmIHRHFl5cqJHeR0lv/m2A5jgVD8Pvy6toTLkubrpUw1AwVmwmyQCC/xmT1CNYqSiZFdNU0EfyD+sy09t9ORy7fr+2c" + "aPZ3qEQcHLCWB5P2vowrKDLzQZt8WL8uCqhMh/XTkw24VGmLaoGyY5uCew6L8eu80x0DKEfaRJRiB64QyDCC2Qkor1yCypqiNNMLI/TenuLI" + "ke83mM7YpHLteXjyjKnxhn7MrfnKO5U590/SH/Kr8mYciXsK51LDcbQYjVo/A2adHAO6d0+H4Vgn6I3fKlnQnQOsPxQApRtqDlZs5hneb4Qb" + "OApYtKzJtZQSyS8yUmAASSpR4Jo/yo6KAtw9TTtmhVs6dWlVidny/yd78X7J0u2ZZYJ4DipDz5WcDeDKc1jasWtFStL3Z39K/X3avQ2X9N6V" + "df+93hUKRBxCvCjT3PDiMv4vcfIjFzIjGA03QfDsRFIrZD3ujCv68CtEiDLkp2Vn29NZdPHTubLn7xgB" +) + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1: Final = ( + "CAQSlgcKEQgSGAI4AUIJbmFycmF0aW9uEgxfkIhrUoVB6AUel5waDEMowcGwIVAmTM40HyIwQifedfhooWeUFo82sSwMDtiDs0UGGwhdjnUL" + "h9QWzjOFDnKlV4106rtLiSqSSLPBKrIG50QLGXmeEfFW/6AT/p1GMwMsw2qCRIMPu6fs23OJ8bT5326+OKflQ9Sl1M5r4EnINcSl5K6h+0vA" + "aikzMcw1vJI16ExuQDc/4J4FXZjGuoKTZN8+VqeAUcJQNAGyZwiQkmV3YUa5kY6rB8+ls7G8/07LWSt/d0OzXR/ax4B5JSHjEiG+kOxPA7IA" + "Z4JDKxtrPYXp5zQ1JrdRXlFEQSe03k6C/HBzpg0njw0cqO23fJIXASmwHA5EV1okl9qqIpIfH2hOQ/v1ufuUkZFzE/jX1nHvzoE0zmvkwGJV" + "11GB6vXzft9CpU6C5PUGYpJUi7+bTObpBSmQrvIVjPaCLZt2fi0U++c7BL8GcNY7B2thHBOhRxns1xz+Psw7z2z4H2+mZmYDTTnSOaq9a3L6" + "FtSdDYcik4QqJg0t7HRnXURvEm7T1r5E5gdVNe28XeBFn+hpKRBY/B4sYlWg2XV/E5EPEX5EfH/noSwSwm7t5olR2bMcynRsjWaeVQC0Mcfq" + "YbtleTJxdm3p1lmBUFZypZfWONK5ZZeSrkU3gZ4LifBta87LHuzl15gWlSPTYlzG6SYIdXytPFI2K7oGyLh/yRs2vjFyZVBhCm5gZ+rkv8JA" + "cAMoTHUoPYgrAotw+dusfSuTr91lULs0wRjTC35K1MOYrb2dUzf0Bh57AXCKE3pCIoXKS8Mq58zHrYXgqzxANFpJQ5P4KzvvQRZa6XbhVgpv" + "kBOkGEemcX57ay9tGYo5fxmHYYihgfJXyEn5dg+xUcwNV7qxprK8f/5piK6TfkvUxAU/Gspg6VUP8f/uHV4JOzZW9s/+l+ImBFbojmKSc5Zz" + "y4HAL3F1E4YlsxkwIkTCs278wCgQnDVIG+UYzny202dxDpdyLaMbLjTWuWzQtRbsg3isYR98ndOLTXgtiJ4N5i5zeqyrbAPsp1xwO4PpmEzg" + "hqNVSJ8vbhL1gBGsVIzSviCPOpL/Z622zhljm5ZhFMzwMqFzzh2PKioNPaVtZgpgn8J6cFRPzI0hCz93pCwRGN25SRcE7VqXdNfGR1k7JxsU" + "ECZAHPt41McAZSpWT63h39oQB54reKKlwhqgw+cQQ0AYAQ==" +) + +RESPONSES_REQUEST: Final[Mapping[str, JsonValue]] = { + "cache": {"no-cache": True}, + "max_output_tokens": 1024, + "instructions": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider zero, " + "negative and non-integer quantities before choosing the next file to inspect.", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + } + ], + }, + { + "role": "assistant", + "content": [{"type": "output_text", "text": "I will locate the quantity validation and its tests."}], + }, + { + "type": "function_call", + "call_id": "toolu_synthetic_search_01", + "name": "search_files", + "arguments": '{"query": "quantity", "paths": ["shop/", "tests/"]}', + }, + { + "type": "function_call_output", + "call_id": "toolu_synthetic_search_01", + "output": '{"matches": [{"path": "shop/validation.py", "line": 18, "text": "def ' + 'validate_quantity(quantity):"}, {"path": "tests/test_validation.py", "line": 7, ' + '"text": "def test_zero_quantity():"}]}', + }, + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Read the validation file first so we can understand how zero and " + "negative quantities are handled.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + ], + }, + ], + "tools": [ + { + "type": "function", + "name": "search_files", + "description": "Search source and test files for a literal term.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + }, + { + "type": "function", + "name": "read_file", + "description": "Read a range of lines from a project file.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + }, + ], + "reasoning": {"effort": "high"}, +} + +RESPONSES_RESPONSE: Final[Mapping[str, JsonValue]] = { + "id": ANY, + "created_at": ANY, + "error": None, + "incomplete_details": None, + "instructions": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the responses client for this project review. Consider zero, " + "negative and non-integer quantities before choosing the next file to inspect.", + "metadata": {}, + "object": "response", + "output": [ + { + "type": "reasoning", + "id": ANY, + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The user wants the validation file read first, but since the test " + "reads are independent, I can just include both in the same batch " + "since it's effectively still prioritizing validation.\n" + "\n" + "I'll check `shop/validation.py` to see how `validate_quantity` " + "handles zero, negative, and non-integer values, and read " + "`tests/test_validation.py` alongside it to confirm the expected " + "behavior.\n" + "\n", + "annotations": [], + } + ], + "phase": None, + "encrypted_content": json.dumps( + [ + { + "type": "thinking", + "thinking": "The user wants the validation file read first, but since the test reads are " + "independent, I can just include both in the same batch since it's effectively still " + "prioritizing validation.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see how `validate_quantity` handles zero, negative, " + "and non-integer values, and read `tests/test_validation.py` alongside it to confirm the " + "expected behavior.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + ], + separators=(",", ":"), + ), + }, + { + "arguments": '{"path": "shop/validation.py", "start_line": 1, "end_line": 60}', + "call_id": "toolu_01GYseiK8hh9QxfWJ6HbiAt3", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01GYseiK8hh9QxfWJ6HbiAt3", + "namespace": None, + "status": "completed", + }, + { + "arguments": '{"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}', + "call_id": "toolu_01U3TuUTifkSBLVzyTFnbMN1", + "name": "read_file", + "type": "function_call", + "id": "fc_toolu_01U3TuUTifkSBLVzyTFnbMN1", + "namespace": None, + "status": "completed", + }, + ], + "parallel_tool_calls": False, + "temperature": None, + "tool_choice": "auto", + "tools": [ + { + "name": "search_files", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "defer_loading": None, + "description": "Search source and test files for a literal term.", + }, + { + "name": "read_file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "strict": None, + "type": "function", + "defer_loading": None, + "description": "Read a range of lines from a project file.", + }, + ], + "top_p": None, + "max_output_tokens": 1024, + "previous_response_id": None, + "reasoning": {"effort": "high"}, + "status": "completed", + "text": {}, + "truncation": None, + "usage": { + "input_tokens": 877, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": 4, + "video_tokens": None, + "cache_write_tokens": 873, + }, + "output_tokens": 509, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 313, "text_tokens": 196}, + "total_tokens": 1386, + "cost": None, + }, + "user": None, + "store": None, + "provider_specific_fields": { + "citations": None, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user wants the validation file " + "read first, but since the test reads " + "are independent, I can just include " + "both in the same batch since it's " + "effectively still prioritizing " + "validation.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "thinking", + "thinking": "I'll check `shop/validation.py` to see " + "how `validate_quantity` handles zero, " + "negative, and non-integer values, and " + "read `tests/test_validation.py` " + "alongside it to confirm the expected " + "behavior.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_1, + }, + ], + }, +} diff --git a/tests/integration/translation/templates/__init__.py b/tests/integration/translation/templates/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/templates/anthropic.py b/tests/integration/translation/templates/anthropic.py new file mode 100644 index 00000000000..e77310c86d1 --- /dev/null +++ b/tests/integration/translation/templates/anthropic.py @@ -0,0 +1,150 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0: Final = ( + "CAQS6AYKEAgSGAI4AUIIdGhpbmtpbmcSDMgHDEwvhCWxzhI8xhoMRhO9WFMe66gmXinaIjCsd5QyXV2yye1/gHsrjZY0DUgMwyexaUDOaW82" + "n43MT0YfpNtKB9eUXLsKedHUC7wqhQaR1+SDqqxoRnT1IqD+0uCRc5ynqXyo5VzPWKVKK6XgabxnRy/oGPw1MPT2cMJbSE+nwGu31Dr8rv58" + "f+ewsNYvFmZ+Pw1H151zoMnrAiSsww7G4XVaMRty5AEQKccViiHRxzNhTlpQhEOZeupQi2c3qfeBBD2zWoww4FQwhQg2SyKZdTuN7vuTnAVW" + "1h9exG2ahsBVEbnJf8Ukp9lEi1YTtays1AzAlnHNj/TTwQvuA0QNRR+652loFhbpkySm1GRlXcgD812rRf1qQqLNGVLzefRQt3k1L3eE2+WB" + "q8vqUBdJkZDxGlK0qSuA4kfacnIihyHO7vfuY/Gg7KAlTX7oDpvV9YbA8ZiFk4C5I/uaNJpeN69S93gutvcJ07YcwtPtHIpCTATQJLY9AQoc" + "gAYacjAGGNGhnzk0v4sOvCcutx1RnXcozOQ6ukG5NNivEK7pW7c+wUpVsTAjH1GPNZHvDmt4FRvLFgwYP9mxrjoD7WTJhd1VdiMZjXxAQVER" + "zFmumgOYZ7/VT+yXTgslC7dgy8fzw3AqgRI+3438BuUNpV7tVf33r7GvnvdTHdK1KGxPaDtacmcxvX8MtX94poRHUsyIAggvH7fu0rCuhzBd" + "/plM9iTizSn3y2mKIGfRm7D9w/B7LwkaPJCHDUSKyOThOGvk4X6XypC45BZbi10mc/nMuLZIDiJ9WWlnxzxON+VLiANxxa3neQ7FSHKPNv45" + "KWW7UzZep+BG3NSp+KPmx2bt4DBj+76HGyrQZs/f6uPBApsoXvnORXGYl945OQbiueIR/Grf2hDKVBijncal/bley82JMptO0N7UI+4aO2Bu" + "OyDuYDy9ysFvd6XahDm6X5/W/tVjCG2TidsKRRSo2hQ9aXRHOjye/8k0SuMdNOJdSg7KQcMqGk2WLxCYFO2BMSftkKW1HmeKQHytB940klJH" + "d9fiq+Yel82rctzObh6pfS3GX7Wax4u//VhERceNh7lrSrailAQVPb3OOu0GR5IcCbiPlPA9/eyVpHyZdwNjiClpSxgB" +) + +ANTHROPIC_MESSAGES_REQUEST: Final[Mapping[str, JsonValue]] = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "The checkout rejects quantities of zero. Find the validation code and its tests.", + } + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "I will locate the quantity validation and its tests."}, + { + "type": "tool_use", + "id": "toolu_synthetic_search_01", + "name": "search_files", + "input": {"query": "quantity", "paths": ["shop/", "tests/"]}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_synthetic_search_01", + "content": '{"matches": [{"path": "shop/validation.py", "line": 18, ' + '"text": "def validate_quantity(quantity):"}, {"path": ' + '"tests/test_validation.py", "line": 7, "text": "def ' + 'test_zero_quantity():"}]}', + }, + { + "type": "text", + "text": "Read the validation file first so we can understand how zero and " + "negative quantities are handled.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + }, + ], + }, + ], + "max_tokens": 1024, + "tools": [ + { + "name": "search_files", + "input_schema": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "paths": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["query", "paths"], + "additionalProperties": False, + }, + "type": "custom", + "description": "Search source and test files for a literal term.", + }, + { + "name": "read_file", + "input_schema": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start_line": {"type": "integer", "minimum": 1}, + "end_line": {"type": "integer", "minimum": 1}, + }, + "required": ["path", "start_line", "end_line"], + "additionalProperties": False, + }, + "type": "custom", + "description": "Read a range of lines from a project file.", + }, + ], + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "high"}, + "system": [ + { + "type": "text", + "text": "You help maintain a synthetic Python project. Read the relevant files before " + "suggesting a fix. Explain which file you need next, then call a read-only tool. " + "Never change files. Use the chat_completions client for this project review. " + "Consider zero, negative and non-integer quantities before choosing the next file " + "to inspect.", + } + ], +} + +ANTHROPIC_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue]] = { + "id": "msg_011Cg32pxq64L8VQGo1JKTnH", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "I'll follow the user's request and read the validation file first, focusing " + "on a reasonable range around line 18.\n" + "\n", + "signature": CLAUDE_SONNET_5_5_THINKING_SIGNATURE_0, + }, + { + "type": "text", + "text": "I need `shop/validation.py` first. It holds `validate_quantity`, which decides " + "how zero, negative and non-integer values are treated. I'll read the area " + "around line 18.", + }, + { + "type": "tool_use", + "id": "toolu_01NdDgeoKnYQe6YXxzc3XDmF", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + ], + "container": None, + "stop_reason": "tool_use", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 876, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 876, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 235, + "output_tokens_details": {"thinking_tokens": 80}, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, +}