From b578dcf278fdf61bb66f3cf28611bd74c7530f8f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:29:56 -0700 Subject: [PATCH] test(translation): recapture anthropic claude bases after usage speed field (#47116) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat_completions/bases/anthropic.py | 58 +++++++---- .../chat_completions/cross_model/anthropic.py | 96 ++++++++++++++++--- .../chat_completions/cross_model/case.py | 9 +- .../translation/messages/bases/anthropic.py | 37 +++++-- .../cross_model/test_messages_cross_model.py | 27 +++++- .../test_messages_service_tier_anthropic.py | 20 +++- .../translation/responses/bases/anthropic.py | 38 +++++--- .../cross_model/test_responses_cross_model.py | 15 ++- 8 files changed, 235 insertions(+), 65 deletions(-) diff --git a/tests/integration/translation/chat_completions/bases/anthropic.py b/tests/integration/translation/chat_completions/bases/anthropic.py index 16e1de53975..3a148c64859 100644 --- a/tests/integration/translation/chat_completions/bases/anthropic.py +++ b/tests/integration/translation/chat_completions/bases/anthropic.py @@ -42,7 +42,7 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-haiku-4-5-20251001", - "id": "msg_011CfgDzBnuPYc7TdysZFRpo", + "id": "msg_011Cg7v1TRmiNAjM2RRxcxF8", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -58,6 +58,7 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( "output_tokens": 5, "service_tier": "standard", "inference_geo": "not_available", + "speed": "standard", }, "diagnostics": None, }, @@ -93,6 +94,7 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "not_available", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -123,7 +125,7 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-sonnet-4-6", - "id": "msg_011CfgDzH3AcaSNXtvurLgAG", + "id": "msg_011Cg7v28wK6VaszM4tL4D6n", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello!"}], @@ -139,6 +141,7 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( "output_tokens": 5, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -174,6 +177,7 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -223,6 +227,7 @@ CLAUDE_FABLE_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -272,6 +277,7 @@ CLAUDE_FABLE_5_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -321,6 +327,7 @@ CLAUDE_OPUS_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -370,6 +377,7 @@ CLAUDE_OPUS_5_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -400,7 +408,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-sonnet-5", - "id": "msg_011CfgDzPkj2HxBq8qGJsGSi", + "id": "msg_011Cg7v2SdRUNPAvGScTF7au", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -417,6 +425,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -452,6 +461,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -482,7 +492,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-opus-4-8", - "id": "msg_011CfgDzV9vZCkuQJt15n3Z1", + "id": "msg_011Cg7v1hPQMJ1QhaC8HaSfQ", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -499,6 +509,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -534,6 +545,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -549,8 +561,16 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( "content-type": "application/json", }, expected_provider_request={**ANTHROPIC_MESSAGES_REQUEST, "model": "claude-sonnet-5-5"}, - mock_provider_response={**ANTHROPIC_MESSAGES_RESPONSE, "model": "claude-sonnet-5-5"}, - expected_litellm_response={**CHAT_COMPLETIONS_RESPONSE, "model": "anthropic/claude-sonnet-5-5"}, + mock_provider_response={ + **ANTHROPIC_MESSAGES_RESPONSE, + "model": "claude-sonnet-5-5", + "usage": {**object_value(ANTHROPIC_MESSAGES_RESPONSE["usage"]), "speed": "standard"}, + }, + expected_litellm_response={ + **CHAT_COMPLETIONS_RESPONSE, + "model": "anthropic/claude-sonnet-5-5", + "usage": {**object_value(CHAT_COMPLETIONS_RESPONSE["usage"]), "speed": "standard"}, + }, ) @@ -618,6 +638,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 149}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, }, expected_litellm_response={ @@ -641,7 +662,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( }, "id": "toolu_015ndrvU4VqqoMBzK35jt4TX", "type": "function", - }, + } ], "reasoning_content": "There's a mention of a chat_completions client I don't actually have access " "to, so I'll ignore that and stick to my available tools. I'll proceed by " @@ -658,7 +679,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "examine next.\n" "\n", "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, - }, + } ], "provider_specific_fields": { "citations": None, @@ -672,11 +693,11 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "deciding what to examine next.\n" "\n", "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, - }, + } ], }, }, - }, + } ], "usage": { "completion_tokens": 288, @@ -694,6 +715,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -757,6 +779,7 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 66}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, }, expected_litellm_response={ @@ -780,7 +803,7 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( }, "id": "toolu_01Ew2u9E9j2mkT6RbwSw13uR", "type": "function", - }, + } ], "reasoning_content": "I'll focus on reading the validation file first, around line 18, since that's " "what was specifically requested before looking at the tests.\n" @@ -793,7 +816,7 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "tests.\n" "\n", "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, - }, + } ], "provider_specific_fields": { "citations": None, @@ -805,11 +828,11 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "the tests.\n" "\n", "signature": CLAUDE_OPUS_5_5_THINKING_SIGNATURE_0, - }, + } ], }, }, - }, + } ], "usage": { "completion_tokens": 215, @@ -827,6 +850,7 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) @@ -869,6 +893,7 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, }, expected_litellm_response={ @@ -892,11 +917,11 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( }, "id": "toolu_01XMSjzRvHCE753Wx4dU2Vxo", "type": "function", - }, + } ], "provider_specific_fields": {"citations": None, "thinking_blocks": None}, }, - }, + } ], "usage": { "completion_tokens": 143, @@ -914,6 +939,7 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "cache_read_input_tokens": 0, "inference_geo": "global", "service_tier": "standard", + "speed": "standard", }, }, ) diff --git a/tests/integration/translation/chat_completions/cross_model/anthropic.py b/tests/integration/translation/chat_completions/cross_model/anthropic.py index ccb543ccf51..a7a06d01361 100644 --- a/tests/integration/translation/chat_completions/cross_model/anthropic.py +++ b/tests/integration/translation/chat_completions/cross_model/anthropic.py @@ -2,10 +2,7 @@ from dataclasses import replace from typing import Final from integration._support.client import list_value, object_value -from integration.translation.case import ( - SSEFrame, - StreamingTranslationTestCase, -) +from integration.translation.case import SSEFrame, StreamingTranslationTestCase from integration.translation.chat_completions.cross_model.contracts import followup_request from integration.translation.chat_completions.bases.anthropic import ( CLAUDE_FABLE_5_1_TEST_CASE, @@ -20,9 +17,84 @@ from integration.translation.chat_completions.templates import ( coding_agent_stream_message, CODING_AGENT_RESPONSE, ) -from integration.translation.templates.anthropic import ( - ANTHROPIC_AGENT_REQUEST, - ANTHROPIC_AGENT_RESPONSE, +from integration.translation.templates.anthropic import ANTHROPIC_AGENT_REQUEST, ANTHROPIC_AGENT_RESPONSE + +CLAUDE_FABLE_5_AGENT_CROSS_MODEL_PROVIDER_USAGE: Final = { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "input_tokens": 1184, + "output_tokens": 105, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", +} +CLAUDE_FABLE_5_AGENT_CROSS_MODEL_CLIENT_USAGE: Final = { + "completion_tokens": 105, + "prompt_tokens": 1184, + "total_tokens": 1289, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1184, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "service_tier": "standard", +} +CLAUDE_FABLE_5_AGENT_CROSS_MODEL_TEST_CASE: Final = replace( + CLAUDE_FABLE_5_TEST_CASE, + mock_provider_response={ + **CLAUDE_FABLE_5_TEST_CASE.mock_provider_response, + "usage": CLAUDE_FABLE_5_AGENT_CROSS_MODEL_PROVIDER_USAGE, + }, + expected_litellm_response={ + **CLAUDE_FABLE_5_TEST_CASE.expected_litellm_response, + "usage": CLAUDE_FABLE_5_AGENT_CROSS_MODEL_CLIENT_USAGE, + }, +) + +CLAUDE_OPUS_5_AGENT_CROSS_MODEL_PROVIDER_USAGE: Final = { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "input_tokens": 1180, + "output_tokens": 105, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", +} +CLAUDE_OPUS_5_AGENT_CROSS_MODEL_CLIENT_USAGE: Final = { + "completion_tokens": 105, + "prompt_tokens": 1180, + "total_tokens": 1285, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1180, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "service_tier": "standard", +} +CLAUDE_OPUS_5_AGENT_CROSS_MODEL_TEST_CASE: Final = replace( + CLAUDE_OPUS_5_TEST_CASE, + mock_provider_response={ + **CLAUDE_OPUS_5_TEST_CASE.mock_provider_response, + "usage": CLAUDE_OPUS_5_AGENT_CROSS_MODEL_PROVIDER_USAGE, + }, + expected_litellm_response={ + **CLAUDE_OPUS_5_TEST_CASE.expected_litellm_response, + "usage": CLAUDE_OPUS_5_AGENT_CROSS_MODEL_CLIENT_USAGE, + }, ) CLAUDE_FABLE_5_1_AGENT_TEST_CASE: Final = replace( @@ -350,7 +422,7 @@ CLAUDE_FABLE_5_1_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase SSEFrame(data={"type": "message_stop"}, event="message_stop"), ), expected_litellm_message=coding_agent_stream_message( - ("toolu_01324ezmdSmGSpmxGX55GvYF", "toolu_01MxHYehH7UYVY2A8WkRJZPu"), + ("toolu_01324ezmdSmGSpmxGX55GvYF", "toolu_01MxHYehH7UYVY2A8WkRJZPu") ), expected_litellm_usage={ "prompt_tokens": 1184, @@ -596,7 +668,7 @@ CLAUDE_FABLE_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( SSEFrame(data={"type": "message_stop"}, event="message_stop"), ), expected_litellm_message=coding_agent_stream_message( - ("toolu_01UXBJZGdCaY4vxon1j4uapy", "toolu_0124RYbAPnwVoGtdzfPCNPc6"), + ("toolu_01UXBJZGdCaY4vxon1j4uapy", "toolu_0124RYbAPnwVoGtdzfPCNPc6") ), expected_litellm_usage={ "prompt_tokens": 1184, @@ -1494,7 +1566,7 @@ CLAUDE_HAIKU_5_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase SSEFrame(data={"type": "message_stop"}, event="message_stop"), ), expected_litellm_message=coding_agent_stream_message( - ("toolu_0177HUb25ArysRAu8eF3u4DJ", "toolu_011gegGWvYoxPwxWY7LSK4Uk"), + ("toolu_0177HUb25ArysRAu8eF3u4DJ", "toolu_011gegGWvYoxPwxWY7LSK4Uk") ), expected_litellm_usage={ "prompt_tokens": 1184, @@ -1837,7 +1909,7 @@ CLAUDE_OPUS_5_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( SSEFrame(data={"type": "message_stop"}, event="message_stop"), ), expected_litellm_message=coding_agent_stream_message( - ("toolu_013mPRCxrKKH6KzYGHKPukwv", "toolu_01FkXiaw4cTiXua3PoMd4wCu"), + ("toolu_013mPRCxrKKH6KzYGHKPukwv", "toolu_01FkXiaw4cTiXua3PoMd4wCu") ), expected_litellm_usage={ "prompt_tokens": 1184, @@ -2088,7 +2160,7 @@ CLAUDE_OPUS_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( SSEFrame(data={"type": "message_stop"}, event="message_stop"), ), expected_litellm_message=coding_agent_stream_message( - ("toolu_017r3PoQsgodP2mgcgDnDLxU", "toolu_01T8ECEb4XwFyozSnCKUiuB2"), + ("toolu_017r3PoQsgodP2mgcgDnDLxU", "toolu_01T8ECEb4XwFyozSnCKUiuB2") ), expected_litellm_usage={ "prompt_tokens": 1180, diff --git a/tests/integration/translation/chat_completions/cross_model/case.py b/tests/integration/translation/chat_completions/cross_model/case.py index d0e842cfa67..07610b398e7 100644 --- a/tests/integration/translation/chat_completions/cross_model/case.py +++ b/tests/integration/translation/chat_completions/cross_model/case.py @@ -1,9 +1,6 @@ from typing import Final -from integration.translation.case import ( - StreamingTranslationTestCase, - TranslationTestCase, -) +from integration.translation.case import StreamingTranslationTestCase, TranslationTestCase from integration.translation.chat_completions.cross_model import ( anthropic, azure, @@ -105,7 +102,7 @@ MID: Final[tuple[AgentCases, ...]] = ( anthropic.CLAUDE_OPUS_5_5_AGENT_STREAMING_TEST_CASE, ), ( - anthropic.CLAUDE_OPUS_5_TEST_CASE, + anthropic.CLAUDE_OPUS_5_AGENT_CROSS_MODEL_TEST_CASE, anthropic.CLAUDE_OPUS_5_AGENT_FOLLOWUP_TEST_CASE, anthropic.CLAUDE_OPUS_5_AGENT_STREAMING_TEST_CASE, ), @@ -189,7 +186,7 @@ HIGH: Final[tuple[AgentCases, ...]] = ( anthropic.CLAUDE_FABLE_5_1_AGENT_STREAMING_TEST_CASE, ), ( - anthropic.CLAUDE_FABLE_5_TEST_CASE, + anthropic.CLAUDE_FABLE_5_AGENT_CROSS_MODEL_TEST_CASE, anthropic.CLAUDE_FABLE_5_AGENT_FOLLOWUP_TEST_CASE, anthropic.CLAUDE_FABLE_5_AGENT_STREAMING_TEST_CASE, ), diff --git a/tests/integration/translation/messages/bases/anthropic.py b/tests/integration/translation/messages/bases/anthropic.py index cc542499a07..be7f487f1b0 100644 --- a/tests/integration/translation/messages/bases/anthropic.py +++ b/tests/integration/translation/messages/bases/anthropic.py @@ -1,6 +1,7 @@ from collections.abc import Mapping from typing import Final +from integration._support.client import object_value from integration.translation.case import TranslationTestCase from integration.translation.messages.templates import MESSAGES_REQUEST, MESSAGES_RESPONSE from integration.translation.templates.anthropic import ANTHROPIC_MESSAGES_REQUEST, ANTHROPIC_MESSAGES_RESPONSE @@ -31,7 +32,7 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-sonnet-4-6", - "id": "msg_011CffzUNHaEfzVxCh5hskBG", + "id": "msg_011Cg7v1w1xZexMrWpf5GU75", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello!"}], @@ -47,12 +48,13 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( "output_tokens": 5, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, expected_litellm_response={ "model": "anthropic/claude-sonnet-4-6", - "id": "msg_011CffzUNHaEfzVxCh5hskBG", + "id": "msg_011Cg7v1w1xZexMrWpf5GU75", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello!"}], @@ -68,6 +70,7 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( "output_tokens": 5, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -98,10 +101,10 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-haiku-4-5-20251001", - "id": "msg_011CfgDz9Vj8wXvKzhoM4MQA", + "id": "msg_011Cg7v1Mt9Eid844XTV8AMc", "type": "message", "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], + "content": [{"type": "text", "text": "Hello!"}], "container": None, "stop_reason": "end_turn", "stop_sequence": None, @@ -114,15 +117,16 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( "output_tokens": 5, "service_tier": "standard", "inference_geo": "not_available", + "speed": "standard", }, "diagnostics": None, }, expected_litellm_response={ "model": "anthropic/claude-haiku-4-5", - "id": "msg_011CfgDz9Vj8wXvKzhoM4MQA", + "id": "msg_011Cg7v1Mt9Eid844XTV8AMc", "type": "message", "role": "assistant", - "content": [{"type": "text", "text": "Hello."}], + "content": [{"type": "text", "text": "Hello!"}], "container": None, "stop_reason": "end_turn", "stop_sequence": None, @@ -135,6 +139,7 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( "output_tokens": 5, "service_tier": "standard", "inference_geo": "not_available", + "speed": "standard", }, "diagnostics": None, }, @@ -165,7 +170,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-sonnet-5", - "id": "msg_011CfgDzLXmJmnHRTpeBoDmZ", + "id": "msg_011Cg7v2KRMCpM2FsiVygFAS", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -182,12 +187,13 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, expected_litellm_response={ "model": "anthropic/claude-sonnet-5", - "id": "msg_011CfgDzLXmJmnHRTpeBoDmZ", + "id": "msg_011Cg7v2KRMCpM2FsiVygFAS", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -204,6 +210,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -234,7 +241,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-opus-4-8", - "id": "msg_011CfgDzSFYYPC3aYE9rH98J", + "id": "msg_011Cg7v1ZPwvewtTtXqjdwzZ", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -251,12 +258,13 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, expected_litellm_response={ "model": "anthropic/claude-opus-4-8", - "id": "msg_011CfgDzSFYYPC3aYE9rH98J", + "id": "msg_011Cg7v1ZPwvewtTtXqjdwzZ", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -273,6 +281,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -377,11 +386,13 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( **ANTHROPIC_MESSAGES_RESPONSE, **CLAUDE_SONNET_5_5_RESPONSE_FIELDS, "model": "claude-sonnet-5-5", + "usage": {**object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, expected_litellm_response={ **MESSAGES_RESPONSE, **CLAUDE_SONNET_5_5_RESPONSE_FIELDS, "model": "anthropic/claude-sonnet-5-5", + "usage": {**object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, ) @@ -460,11 +471,13 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( **ANTHROPIC_MESSAGES_RESPONSE, **CLAUDE_HAIKU_5_5_RESPONSE_FIELDS, "model": "claude-haiku-5-5", + "usage": {**object_value(CLAUDE_HAIKU_5_5_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, expected_litellm_response={ **MESSAGES_RESPONSE, **CLAUDE_HAIKU_5_5_RESPONSE_FIELDS, "model": "anthropic/claude-haiku-5-5", + "usage": {**object_value(CLAUDE_HAIKU_5_5_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, ) @@ -555,11 +568,13 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( **ANTHROPIC_MESSAGES_RESPONSE, **CLAUDE_OPUS_5_5_RESPONSE_FIELDS, "model": "claude-opus-5-5", + "usage": {**object_value(CLAUDE_OPUS_5_5_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, expected_litellm_response={ **MESSAGES_RESPONSE, **CLAUDE_OPUS_5_5_RESPONSE_FIELDS, "model": "anthropic/claude-opus-5-5", + "usage": {**object_value(CLAUDE_OPUS_5_5_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, ) @@ -651,10 +666,12 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( **ANTHROPIC_MESSAGES_RESPONSE, **CLAUDE_FABLE_5_1_RESPONSE_FIELDS, "model": "claude-fable-5-1", + "usage": {**object_value(CLAUDE_FABLE_5_1_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, expected_litellm_response={ **MESSAGES_RESPONSE, **CLAUDE_FABLE_5_1_RESPONSE_FIELDS, "model": "anthropic/claude-fable-5-1", + "usage": {**object_value(CLAUDE_FABLE_5_1_RESPONSE_FIELDS["usage"]), "speed": "standard"}, }, ) diff --git a/tests/integration/translation/messages/cross_model/test_messages_cross_model.py b/tests/integration/translation/messages/cross_model/test_messages_cross_model.py index ae6fe70fe7c..e3bafc37993 100644 --- a/tests/integration/translation/messages/cross_model/test_messages_cross_model.py +++ b/tests/integration/translation/messages/cross_model/test_messages_cross_model.py @@ -3,11 +3,14 @@ from dataclasses import replace from typing import Final import pytest -from integration._support.client import JSON_OBJECT, Gateway +from integration._support.client import JSON_OBJECT, Gateway, object_value from integration._support.provider import SharedProvider from integration._support.wire import Reply from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.messages.bases.anthropic import ( + CLAUDE_SONNET_5_5_RESPONSE_FIELDS, + CLAUDE_SONNET_5_5_TEST_CASE, +) from integration.translation.messages.bases.azure_ai import ( CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE, ) @@ -17,6 +20,18 @@ from integration.translation.response_scenarios import text_completion, two_mess from integration.translation.runner import assert_provider_request, assert_translation GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE) +CLAUDE_SONNET_5_5_CROSS_MODEL_NATIVE_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_TEST_CASE, + scenario="cross_model_rich_request", + mock_provider_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), + }, + expected_litellm_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, + "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), + }, +) AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( AZURE_CLAUDE_SONNET_5_5_TEST_CASE, scenario="cross_model_rich_request", @@ -25,10 +40,14 @@ AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( **CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, "system": [{"type": "text", "text": CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request["system"]}], }, - mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + mock_provider_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), + }, expected_litellm_response={ **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, "model": "azure_ai/claude-sonnet-5-5", + "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), }, ) @@ -44,7 +63,7 @@ def test_messages_keep_the_same_public_contract_on_native_and_bridged_routes( assert {key: value for key, value in case.litellm_request.items() if key != "model"} == { key: value for key, value in CLAUDE_SONNET_5_5_TEST_CASE.litellm_request.items() if key != "model" } - native: Final = assert_translation(CLAUDE_SONNET_5_5_TEST_CASE, gateway, provider) + native: Final = assert_translation(CLAUDE_SONNET_5_5_CROSS_MODEL_NATIVE_TEST_CASE, gateway, provider) compared: Final = assert_translation(case, gateway, provider) assert messages_structure(compared) == messages_structure(native) diff --git a/tests/integration/translation/messages/service_tier/test_messages_service_tier_anthropic.py b/tests/integration/translation/messages/service_tier/test_messages_service_tier_anthropic.py index 5a3a760ffdc..6d7959b5de5 100644 --- a/tests/integration/translation/messages/service_tier/test_messages_service_tier_anthropic.py +++ b/tests/integration/translation/messages/service_tier/test_messages_service_tier_anthropic.py @@ -8,6 +8,19 @@ from integration.translation.case import TranslationTestCase from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE from integration.translation.runner import assert_translation +CLAUDE_SONNET_4_6_SERVICE_TIER_RESPONSE_FIELDS: Final = { + "content": [{"type": "text", "text": "Hello!"}], + "usage": { + "input_tokens": 18, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 5, + "service_tier": "standard", + "inference_geo": "global", + }, +} + CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE: Final = replace( CLAUDE_SONNET_4_6_TEST_CASE, scenario="service_tier", @@ -16,9 +29,14 @@ CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE: Final = replace( **CLAUDE_SONNET_4_6_TEST_CASE.expected_provider_request, "service_tier": "standard_only", }, - mock_provider_response={**CLAUDE_SONNET_4_6_TEST_CASE.mock_provider_response, "id": "msg_011Cg1RG36pCcTLE26htFFqC"}, + mock_provider_response={ + **CLAUDE_SONNET_4_6_TEST_CASE.mock_provider_response, + **CLAUDE_SONNET_4_6_SERVICE_TIER_RESPONSE_FIELDS, + "id": "msg_011Cg1RG36pCcTLE26htFFqC", + }, expected_litellm_response={ **CLAUDE_SONNET_4_6_TEST_CASE.expected_litellm_response, + **CLAUDE_SONNET_4_6_SERVICE_TIER_RESPONSE_FIELDS, "id": "msg_011Cg1RG36pCcTLE26htFFqC", }, ) diff --git a/tests/integration/translation/responses/bases/anthropic.py b/tests/integration/translation/responses/bases/anthropic.py index e30c1f15699..8d5051acb92 100644 --- a/tests/integration/translation/responses/bases/anthropic.py +++ b/tests/integration/translation/responses/bases/anthropic.py @@ -35,7 +35,7 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-haiku-4-5-20251001", - "id": "msg_011CfgDzBnuPYc7TdysZFRpo", + "id": "msg_011Cg7v1CTrRytLeZfaZojBR", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -51,6 +51,7 @@ CLAUDE_HAIKU_4_5_TEST_CASE: Final = TranslationTestCase( "output_tokens": 5, "service_tier": "standard", "inference_geo": "not_available", + "speed": "standard", }, "diagnostics": None, }, @@ -130,10 +131,10 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-sonnet-4-6", - "id": "msg_011CfgDzH3AcaSNXtvurLgAG", + "id": "msg_011Cg7v1kr1sYXyQA143nsda", "type": "message", "role": "assistant", - "content": [{"type": "text", "text": "Hello!"}], + "content": [{"type": "text", "text": "Hello! How can I help you?"}], "container": None, "stop_reason": "end_turn", "stop_sequence": None, @@ -143,9 +144,10 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 5, + "output_tokens": 11, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -164,7 +166,7 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( "id": ANY, "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "Hello!", "annotations": []}], + "content": [{"type": "output_text", "text": "Hello! How can I help you?", "annotations": []}], "phase": None, } ], @@ -190,9 +192,9 @@ CLAUDE_SONNET_4_6_TEST_CASE: Final = TranslationTestCase( "video_tokens": None, "cache_write_tokens": 0, }, - "output_tokens": 5, - "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 5}, - "total_tokens": 23, + "output_tokens": 11, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": 11}, + "total_tokens": 29, "cost": None, }, "user": None, @@ -225,7 +227,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-sonnet-5", - "id": "msg_011CfgDzPkj2HxBq8qGJsGSi", + "id": "msg_011Cg7v215mkRTDBRoPcerLf", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -242,6 +244,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -321,7 +324,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( }, mock_provider_response={ "model": "claude-opus-4-8", - "id": "msg_011CfgDzV9vZCkuQJt15n3Z1", + "id": "msg_011Cg7v1PgnYbGrw8eGpka6f", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello."}], @@ -338,6 +341,7 @@ CLAUDE_OPUS_4_8_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, "diagnostics": None, }, @@ -462,6 +466,7 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 313}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, }, expected_litellm_response={**RESPONSES_RESPONSE, "model": "anthropic/claude-sonnet-5-5"}, @@ -534,6 +539,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 67}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, }, expected_litellm_response={ @@ -553,7 +559,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "requested.\n" "\n", "annotations": [], - }, + } ], "phase": None, "encrypted_content": json.dumps( @@ -565,7 +571,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "requested.\n" "\n", "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, - }, + } ], separators=(",", ":"), ), @@ -605,7 +611,7 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( "range near the top of the file since no specific lines were requested.\n" "\n", "signature": CLAUDE_HAIKU_5_5_THINKING_SIGNATURE_0, - }, + } ], }, }, @@ -698,6 +704,7 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 130}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, }, expected_litellm_response={ @@ -719,7 +726,7 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( "zero, negative, and non-integer inputs before moving to the test file.\n" "\n", "annotations": [], - }, + } ], "phase": None, "encrypted_content": json.dumps( @@ -842,6 +849,7 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "output_tokens_details": {"thinking_tokens": 0}, "service_tier": "standard", "inference_geo": "global", + "speed": "standard", }, }, expected_litellm_response={ @@ -860,7 +868,7 @@ CLAUDE_FABLE_5_1_TEST_CASE: Final = TranslationTestCase( "rejection, then I'll read the test file in a separate step since there's " "nothing to derive from it for the read.", "annotations": [], - }, + } ], "phase": None, }, diff --git a/tests/integration/translation/responses/cross_model/test_responses_cross_model.py b/tests/integration/translation/responses/cross_model/test_responses_cross_model.py index cb6f973d2df..a0b5778a200 100644 --- a/tests/integration/translation/responses/cross_model/test_responses_cross_model.py +++ b/tests/integration/translation/responses/cross_model/test_responses_cross_model.py @@ -22,12 +22,25 @@ from integration.translation.responses.bases.openai_responses import GPT_5_6_SOL from integration.translation.runner import assert_translation GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE) +CLAUDE_SONNET_5_5_CROSS_MODEL_PROVIDER_USAGE: Final = { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 509, + "output_tokens_details": {"thinking_tokens": 313}, + "service_tier": "standard", + "inference_geo": "global", +} AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( AZURE_CLAUDE_SONNET_5_5_TEST_CASE, scenario="cross_model_rich_request", litellm_request={**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, "model": "azure_ai/claude-sonnet-5-5"}, expected_provider_request=CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, - mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + mock_provider_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + "usage": CLAUDE_SONNET_5_5_CROSS_MODEL_PROVIDER_USAGE, + }, expected_litellm_response={ **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, "model": "azure_ai/claude-sonnet-5-5",