From fa716ec7c646a57466fac32404618af3c50bf2dc Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:15:05 +0000 Subject: [PATCH] test(translation): allow anthropic usage speed in cross-model contract (#47130) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat_completions/cross_model/anthropic.py | 78 ------------------- .../chat_completions/cross_model/azure_ai.py | 35 --------- .../chat_completions/cross_model/case.py | 6 +- .../chat_completions/cross_model/contracts.py | 4 +- .../cross_model/test_messages_cross_model.py | 27 +------ .../translation/response_contracts.py | 1 + .../cross_model/test_responses_cross_model.py | 15 +--- 7 files changed, 12 insertions(+), 154 deletions(-) diff --git a/tests/integration/translation/chat_completions/cross_model/anthropic.py b/tests/integration/translation/chat_completions/cross_model/anthropic.py index a7a06d01361..3df7da17ef6 100644 --- a/tests/integration/translation/chat_completions/cross_model/anthropic.py +++ b/tests/integration/translation/chat_completions/cross_model/anthropic.py @@ -19,84 +19,6 @@ from integration.translation.chat_completions.templates import ( ) from integration.translation.templates.anthropic import ANTHROPIC_AGENT_REQUEST, ANTHROPIC_AGENT_RESPONSE -CLAUDE_FABLE_5_AGENT_CROSS_MODEL_PROVIDER_USAGE: Final = { - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "input_tokens": 1184, - "output_tokens": 105, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", -} -CLAUDE_FABLE_5_AGENT_CROSS_MODEL_CLIENT_USAGE: Final = { - "completion_tokens": 105, - "prompt_tokens": 1184, - "total_tokens": 1289, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 1184, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "inference_geo": "global", - "service_tier": "standard", -} -CLAUDE_FABLE_5_AGENT_CROSS_MODEL_TEST_CASE: Final = replace( - CLAUDE_FABLE_5_TEST_CASE, - mock_provider_response={ - **CLAUDE_FABLE_5_TEST_CASE.mock_provider_response, - "usage": CLAUDE_FABLE_5_AGENT_CROSS_MODEL_PROVIDER_USAGE, - }, - expected_litellm_response={ - **CLAUDE_FABLE_5_TEST_CASE.expected_litellm_response, - "usage": CLAUDE_FABLE_5_AGENT_CROSS_MODEL_CLIENT_USAGE, - }, -) - -CLAUDE_OPUS_5_AGENT_CROSS_MODEL_PROVIDER_USAGE: Final = { - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - "input_tokens": 1180, - "output_tokens": 105, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "global", -} -CLAUDE_OPUS_5_AGENT_CROSS_MODEL_CLIENT_USAGE: Final = { - "completion_tokens": 105, - "prompt_tokens": 1180, - "total_tokens": 1285, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 1180, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "inference_geo": "global", - "service_tier": "standard", -} -CLAUDE_OPUS_5_AGENT_CROSS_MODEL_TEST_CASE: Final = replace( - CLAUDE_OPUS_5_TEST_CASE, - mock_provider_response={ - **CLAUDE_OPUS_5_TEST_CASE.mock_provider_response, - "usage": CLAUDE_OPUS_5_AGENT_CROSS_MODEL_PROVIDER_USAGE, - }, - expected_litellm_response={ - **CLAUDE_OPUS_5_TEST_CASE.expected_litellm_response, - "usage": CLAUDE_OPUS_5_AGENT_CROSS_MODEL_CLIENT_USAGE, - }, -) - CLAUDE_FABLE_5_1_AGENT_TEST_CASE: Final = replace( CLAUDE_FABLE_5_1_TEST_CASE, scenario="agent_tools", diff --git a/tests/integration/translation/chat_completions/cross_model/azure_ai.py b/tests/integration/translation/chat_completions/cross_model/azure_ai.py index 15cbe4a3200..73d0ffbb8df 100644 --- a/tests/integration/translation/chat_completions/cross_model/azure_ai.py +++ b/tests/integration/translation/chat_completions/cross_model/azure_ai.py @@ -17,41 +17,6 @@ from integration.translation.chat_completions.templates import ( ) from integration.translation.templates.anthropic import ANTHROPIC_AGENT_REQUEST, ANTHROPIC_AGENT_RESPONSE -CLAUDE_FABLE_5_CROSS_MODEL_TEST_CASE: Final = replace( - CLAUDE_FABLE_5_TEST_CASE, - mock_provider_response={ - **CLAUDE_FABLE_5_TEST_CASE.mock_provider_response, - "usage": { - **object_value(ANTHROPIC_AGENT_RESPONSE["usage"]), - "input_tokens": 1184, - "output_tokens": 105, - "output_tokens_details": {"thinking_tokens": 0}, - "service_tier": "standard", - "inference_geo": "not_available", - }, - }, - expected_litellm_response={ - **CLAUDE_FABLE_5_TEST_CASE.expected_litellm_response, - "usage": { - "completion_tokens": 105, - "prompt_tokens": 1184, - "total_tokens": 1289, - "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 1184, - "cache_write_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "inference_geo": "not_available", - "service_tier": "standard", - }, - }, -) - CLAUDE_FABLE_5_1_AGENT_TEST_CASE: Final = replace( CLAUDE_FABLE_5_1_TEST_CASE, scenario="agent_tools", diff --git a/tests/integration/translation/chat_completions/cross_model/case.py b/tests/integration/translation/chat_completions/cross_model/case.py index fae659c5dca..d81e6a269f6 100644 --- a/tests/integration/translation/chat_completions/cross_model/case.py +++ b/tests/integration/translation/chat_completions/cross_model/case.py @@ -102,7 +102,7 @@ MID: Final[tuple[AgentCases, ...]] = ( anthropic.CLAUDE_OPUS_5_5_AGENT_STREAMING_TEST_CASE, ), ( - anthropic.CLAUDE_OPUS_5_AGENT_CROSS_MODEL_TEST_CASE, + anthropic.CLAUDE_OPUS_5_TEST_CASE, anthropic.CLAUDE_OPUS_5_AGENT_FOLLOWUP_TEST_CASE, anthropic.CLAUDE_OPUS_5_AGENT_STREAMING_TEST_CASE, ), @@ -201,7 +201,7 @@ HIGH: Final[tuple[AgentCases, ...]] = ( anthropic.CLAUDE_FABLE_5_1_AGENT_STREAMING_TEST_CASE, ), ( - anthropic.CLAUDE_FABLE_5_AGENT_CROSS_MODEL_TEST_CASE, + anthropic.CLAUDE_FABLE_5_TEST_CASE, anthropic.CLAUDE_FABLE_5_AGENT_FOLLOWUP_TEST_CASE, anthropic.CLAUDE_FABLE_5_AGENT_STREAMING_TEST_CASE, ), @@ -211,7 +211,7 @@ HIGH: Final[tuple[AgentCases, ...]] = ( azure_ai.CLAUDE_FABLE_5_1_AGENT_STREAMING_TEST_CASE, ), ( - azure_ai.CLAUDE_FABLE_5_CROSS_MODEL_TEST_CASE, + azure_ai.CLAUDE_FABLE_5_TEST_CASE, azure_ai.CLAUDE_FABLE_5_AGENT_FOLLOWUP_TEST_CASE, azure_ai.CLAUDE_FABLE_5_AGENT_STREAMING_TEST_CASE, ), diff --git a/tests/integration/translation/chat_completions/cross_model/contracts.py b/tests/integration/translation/chat_completions/cross_model/contracts.py index 4fbf6a880b5..a15a43985f1 100644 --- a/tests/integration/translation/chat_completions/cross_model/contracts.py +++ b/tests/integration/translation/chat_completions/cross_model/contracts.py @@ -121,7 +121,9 @@ PROVIDER_EXTENSIONS: Final = MappingProxyType( ("provider_specific_fields", "reasoning_content", "thinking_blocks", "reasoning_items", "images") ), "choices[].message.tool_calls[]": frozenset(("index", "caller", "provider_specific_fields")), - "usage": frozenset(("cache_creation_input_tokens", "cache_read_input_tokens", "inference_geo", "service_tier")), + "usage": frozenset( + ("cache_creation_input_tokens", "cache_read_input_tokens", "inference_geo", "service_tier", "speed") + ), } ) diff --git a/tests/integration/translation/messages/cross_model/test_messages_cross_model.py b/tests/integration/translation/messages/cross_model/test_messages_cross_model.py index e3bafc37993..ae6fe70fe7c 100644 --- a/tests/integration/translation/messages/cross_model/test_messages_cross_model.py +++ b/tests/integration/translation/messages/cross_model/test_messages_cross_model.py @@ -3,14 +3,11 @@ from dataclasses import replace from typing import Final import pytest -from integration._support.client import JSON_OBJECT, Gateway, object_value +from integration._support.client import JSON_OBJECT, Gateway from integration._support.provider import SharedProvider from integration._support.wire import Reply from integration.translation.case import TranslationTestCase -from integration.translation.messages.bases.anthropic import ( - CLAUDE_SONNET_5_5_RESPONSE_FIELDS, - CLAUDE_SONNET_5_5_TEST_CASE, -) +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE from integration.translation.messages.bases.azure_ai import ( CLAUDE_SONNET_5_5_TEST_CASE as AZURE_CLAUDE_SONNET_5_5_TEST_CASE, ) @@ -20,18 +17,6 @@ from integration.translation.response_scenarios import text_completion, two_mess from integration.translation.runner import assert_provider_request, assert_translation GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE) -CLAUDE_SONNET_5_5_CROSS_MODEL_NATIVE_TEST_CASE: Final = replace( - CLAUDE_SONNET_5_5_TEST_CASE, - scenario="cross_model_rich_request", - mock_provider_response={ - **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, - "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), - }, - expected_litellm_response={ - **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, - "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), - }, -) AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( AZURE_CLAUDE_SONNET_5_5_TEST_CASE, scenario="cross_model_rich_request", @@ -40,14 +25,10 @@ AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( **CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, "system": [{"type": "text", "text": CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request["system"]}], }, - mock_provider_response={ - **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, - "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), - }, + mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, expected_litellm_response={ **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, "model": "azure_ai/claude-sonnet-5-5", - "usage": object_value(CLAUDE_SONNET_5_5_RESPONSE_FIELDS["usage"]), }, ) @@ -63,7 +44,7 @@ def test_messages_keep_the_same_public_contract_on_native_and_bridged_routes( assert {key: value for key, value in case.litellm_request.items() if key != "model"} == { key: value for key, value in CLAUDE_SONNET_5_5_TEST_CASE.litellm_request.items() if key != "model" } - native: Final = assert_translation(CLAUDE_SONNET_5_5_CROSS_MODEL_NATIVE_TEST_CASE, gateway, provider) + native: Final = assert_translation(CLAUDE_SONNET_5_5_TEST_CASE, gateway, provider) compared: Final = assert_translation(case, gateway, provider) assert messages_structure(compared) == messages_structure(native) diff --git a/tests/integration/translation/response_contracts.py b/tests/integration/translation/response_contracts.py index c3182d269ff..6af3f9d681d 100644 --- a/tests/integration/translation/response_contracts.py +++ b/tests/integration/translation/response_contracts.py @@ -197,6 +197,7 @@ MESSAGES_OPTIONAL: Final = MappingProxyType( "server_tool_use", "service_tier", "inference_geo", + "speed", ) ), } diff --git a/tests/integration/translation/responses/cross_model/test_responses_cross_model.py b/tests/integration/translation/responses/cross_model/test_responses_cross_model.py index a0b5778a200..cb6f973d2df 100644 --- a/tests/integration/translation/responses/cross_model/test_responses_cross_model.py +++ b/tests/integration/translation/responses/cross_model/test_responses_cross_model.py @@ -22,25 +22,12 @@ from integration.translation.responses.bases.openai_responses import GPT_5_6_SOL from integration.translation.runner import assert_translation GPT_6_1_SOL_TWO_MESSAGES_BEFORE_TOOLS_TEST_CASE: Final = two_messages_before_tools(GPT_6_1_SOL_TEST_CASE) -CLAUDE_SONNET_5_5_CROSS_MODEL_PROVIDER_USAGE: Final = { - "input_tokens": 4, - "cache_creation_input_tokens": 873, - "cache_read_input_tokens": 0, - "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, - "output_tokens": 509, - "output_tokens_details": {"thinking_tokens": 313}, - "service_tier": "standard", - "inference_geo": "global", -} AZURE_CLAUDE_SONNET_5_5_CROSS_MODEL_TEST_CASE: Final = replace( AZURE_CLAUDE_SONNET_5_5_TEST_CASE, scenario="cross_model_rich_request", litellm_request={**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, "model": "azure_ai/claude-sonnet-5-5"}, expected_provider_request=CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, - mock_provider_response={ - **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, - "usage": CLAUDE_SONNET_5_5_CROSS_MODEL_PROVIDER_USAGE, - }, + mock_provider_response=CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, expected_litellm_response={ **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, "model": "azure_ai/claude-sonnet-5-5",