diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index c552e11c22a..f1e44930aab 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -326,6 +326,11 @@ model_list: model: openai/gpt-5.6-terra api_base: http://127.0.0.1:8191 api_key: synthetic-openai-key + - model_name: openai/gpt-6.1-terra + litellm_params: + model: openai/gpt-6.1-terra + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key - model_name: openai/responses/gpt-5.6-sol litellm_params: model: openai/responses/gpt-5.6-sol diff --git a/tests/integration/translation/chat_completions/bases/openai.py b/tests/integration/translation/chat_completions/bases/openai.py index 25d8121cf31..355e01be180 100644 --- a/tests/integration/translation/chat_completions/bases/openai.py +++ b/tests/integration/translation/chat_completions/bases/openai.py @@ -451,6 +451,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( ) + GPT_5_MINI_TEST_CASE: Final = TranslationTestCase( scenario="agent_tools", litellm_endpoint="/v1/chat/completions", @@ -1235,3 +1236,276 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( }, }, ) + + +GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE: Final = { + 'id': 'resp_0b85038a4a37efb6006ae75c0cf84c87d1ac49e75264e5701f', + 'object': 'response', + 'created_at': 1793547277, + 'status': 'completed', + 'access_programs': { + 'cyber': 'standard', + }, + 'background': False, + 'billing': { + 'payer': 'developer', + }, + 'completed_at': 1793547279, + 'error': None, + 'frequency_penalty': 0.0, + 'incomplete_details': None, + 'instructions': ( + 'You help maintain a synthetic Python project. Read the relevant files before suggesting ' + 'a fix. Explain which file you need next, then call a read-only tool. Never change files.' + ' Use the chat_completions client for this project review. Consider zero, negative and no' + 'n-integer quantities before choosing the next file to inspect.' + ), + 'max_output_tokens': 1024, + 'max_tool_calls': None, + 'model': 'gpt-6.1-terra', + 'moderation': None, + 'output': [ + { + 'id': 'rs_0b85038a4a37efb6006ae75c0de1ac87d1a40d2ce0476272d9', + 'type': 'reasoning', + 'content': [], + 'encrypted_content': ( + 'gAAAAABq51wPWhH43zwDEF4VgA1Kgnfi13a_k7Ff26kP_ipp7PFCxc9W_3tlMScFQVruAcFY4-QJk3WYSiz_ErCS' + 'n7_u9vSRN1KB6ytyA4CMads8YIpqomBsURrtDZKtt26D2sp9DgthN5hg4sMsAiuOGkhNWwQ91cXOv6YXmKRQ3nTX' + 'ZhMXy07iuYELaE10_tszlQ6ZG0Etml2Ka59Y_Rod-NQHRtWSfowQrqhaYgwDimAMfuVdOo4myHtwIAWT8O4I2oAH' + 'BwXm2LcKPUh7DcUBNvdrqTSO9YfgFpmc82-y5xaHYcRd2nM0s5_AaQfISy1LANBI7l5yY-D_FxxISG6zyzOagbLO' + 'lLhdP3ftLpwAqu_xp13sJELtUR0r3itiDRxllGU9r0oIQ3b8bu3Z0bOH-1WWRBTEBUVCeDSadRIpa-gRJhZ-DSir' + 'wvZbqpZrKhn7LtaW0wgQlwuUZUCGk7MxdTrKe7Zw7K86iXY39outrZ4JACMUx5bez-_O0WVyuvjeIWAHKCSH9ofj' + 'bZXjPpG6mKaIggIGw8nZBr_uy60TVTvWvodJPoJ50lEDoi0ZClDgtQol7ypWSKBIVViuNMgHp27n4BFlGfl57Aef' + 'nD3PzqHFUNCawiH56wP1YgcsrwnlFCPKkFfgZHTGj190Rg8gTFRpOYfNLrI65AIR6LrUY091yqwAfifk3iyh5qy7' + 'YGtNyYw0IGTV3OYKoQJWCi3Pct6f9tNUenTtq7e-UT0OxVQxe_yyBPTm73TthwygwTSVczQ25q49Buzu7Dk7YWAw' + 'AvKGfvT9dqn7MsAV37PjBYTeRjZr23uNHIyPc8wHx5gq_1czCRDASUx6rEhQQzcvhG79ru2_apH61JWAYiVnfBAT' + 'zsvfabELZ9xdZxiY_p30L4AQKaV4MGPfYuqlKWpY3mxRUGjqres5KqVd5-rGWsfm4xlPYQpRnnKcNbEd1gNfDdc2' + 'IQUBmRB9zMVszhvWTFleLJrd84ztACnuC8DKxFSxihZ8QDKJbIEMqpYqm6ZUCljweUI--f9nYaoX9OIl5ch4tPoq' + '7SVRdG1pxfAo7RubrgAYNiACntgbhpF8Wf9wWbXTxgBco4QvtdeEqEm6Fp--fK5uMvf2aUXL8mCda8Z-KZBWlO1v' + '4xh8cHRJOkIU0rWNLYaPpmmEq109oL6B3gmSe7xrkAzhsD5ITBziXgZZvxY4r34JfLlBWRrx_1sEqFNTCtuaibVZ' + 'ZK15AysbTmMlumjfkP-tC8ygdtqJfSUUsVFzxgf8z6ThoP0MeMeuOKA79VeHpCnl758U80BIeUicDBBwwYIP8kpU' + 'ZyGqUTZcxDNQ80U9mXYMErrDf0PnzisTpw2Tm6gytBdjKfH07P272VfNJYHsO53l7oOkyUS1e9OdeWBEMaDje1B0' + 'QITl4-laHjbXnMIdZUdsxWTTiCljzzQQVoN8nQASBTvw-QJNFMqjAyBOt02bvCOVh7Jb9VJrTpXmVZKZ2HDHccma' + '7EohOL0qklhQ6EXxPdn_Gs3-9pGESbfOFZaf01A6gH7iznMxRUMVYAUl4yqr8PFGpA-uEljFhScorVtN1Rm5QUGA' + 'qiCMUnVjeb40_swu8wlC5y7liGJXEiq0zo0wgrn_HU3FZXmWXbBHAkCWlFt8pz2CPIMNLUvc4HaN8Xlcq6kK2aX5' + 'OH_Q8tVUsIS_0SpoAjWVc-6V9a7veqoik2yQyujvjw0b9sESo9qyZeiYj0M73v6SqBfjW2ACTZk5v9V5xLYgjOxZ' + 'Pe-XALR50X3B0FEalWUkUC-uv0scJcHYxb8k2n21OPr4F0Lb01gdTFyk9cbfXyJH5WCc-TGYlmCk3izsNRrWyYNo' + 'g8y77dkxKMjQRGrut-bU6jpu6ctDfOcgTVC-q6sKXURqfNQfP_1ZRCmDApKn3tuTt0vVahZOIJF3rGjvy5bzjyWA' + 'LBqxnsMxeLEEZnjfzqwG6ElioJjN5tHf4MJFV3I4aCmtSOGsaWfdF8_sYSwZ48Q16UO_-BzE_dE2W2wOQZ2inbut' + 'IhUU3r-ga_ogGlntLMJ2GD2r2OgCHJ59PHgL-nOnrbfQdZ_sXmcovUnQiAjKuFAx48cQEZ-i74QAbaCgmUFdrDue' + '_oWlk76IWXR15mwWLbpBZMR5aG4WREsvroJ17mIabFGi7R2_f7Vpe401W04gqDQjgxhkN0zY4rwBCmWZg6YJNntz' + 'qgIWoOcc9ZKUEOh-bpyhoWIyFIgtm4YlQQT-nDJwfRt7CCAz9LwTlx8czHn7JoIR4Oj6-fFY06ILYPmn8TOFFCQb' + 'ZcbdfY-NCE9TU1G60AHddQllarXRSxL4J-MN9tQhk22Ib-cPpGgtSL3L2-12zs0O3SiNUisxdaEEUZpaxauuAu38' + 'TH1_pelomrwvCdE1ukyyQBKDGJkxf4PxUy298fsg0-RQgALmaKa_ZWl68o5weQyPz-phC9GLjwUCgaKzijoI3fec' + 'Low1EDYkbCmkGFJhBiaAsl-AexuSKgFwKvTAg-O1R4SYPRShr6laHfzP9A5VC2hrcQ==' + ), + 'summary': [], + }, + { + 'id': 'msg_0b85038a4a37efb6006ae75c0ec8b887d1b503c5d0835acef1', + 'type': 'message', + 'status': 'completed', + 'content': [ + { + 'type': 'output_text', + 'annotations': [], + 'logprobs': [], + 'text': ( + 'I’ll read `shop/validation.py` first to check how it handles zero, negative, and non-int' + 'eger quantities.' + ), + }, + ], + 'phase': 'commentary', + 'role': 'assistant', + }, + { + 'id': 'fc_0b85038a4a37efb6006ae75c0f3af487d1ade6d1cb6b7aab90', + 'type': 'function_call', + 'status': 'completed', + 'arguments': '{"path":"shop/validation.py","start_line":1,"end_line":200}', + 'call_id': 'call_AJa3dZ4npr0KPN0SEwwl90xc', + 'name': 'read_file', + }, + ], + 'parallel_tool_calls': True, + 'presence_penalty': 0.0, + 'previous_response_id': None, + 'prompt_cache_key': None, + 'prompt_cache_retention': '24h', + 'reasoning': { + 'context': 'all_turns', + 'effort': 'high', + 'mode': 'standard', + 'summary': None, + }, + 'safety_identifier': None, + 'service_tier': 'default', + 'store': True, + 'temperature': 1.0, + 'text': { + 'format': { + 'type': 'text', + }, + 'verbosity': 'medium', + }, + 'tool_choice': 'auto', + 'tool_usage': { + 'image_gen': { + 'input_tokens': 0, + 'input_tokens_details': { + 'image_tokens': 0, + 'text_tokens': 0, + }, + 'output_tokens': 0, + 'output_tokens_details': { + 'image_tokens': 0, + 'text_tokens': 0, + }, + 'total_tokens': 0, + }, + 'web_search': { + 'num_requests': 0, + }, + }, + 'tools': [ + { + 'type': 'function', + 'description': 'Search source and test files for a literal term.', + 'name': 'search_files', + 'output_schema': None, + 'parameters': { + 'type': 'object', + 'properties': { + 'query': { + 'type': 'string', + }, + 'paths': { + 'type': 'array', + 'items': { + 'type': 'string', + }, + }, + }, + 'required': [ + 'query', + 'paths', + ], + 'additionalProperties': False, + }, + 'strict': False, + }, + { + 'type': 'function', + 'description': 'Read a range of lines from a project file.', + 'name': 'read_file', + 'output_schema': None, + 'parameters': { + 'type': 'object', + 'properties': { + 'path': { + 'type': 'string', + }, + 'start_line': { + 'type': 'integer', + 'minimum': 1, + }, + 'end_line': { + 'type': 'integer', + 'minimum': 1, + }, + }, + 'required': [ + 'path', + 'start_line', + 'end_line', + ], + 'additionalProperties': False, + }, + 'strict': False, + }, + ], + 'top_logprobs': 0, + 'top_p': 0.98, + 'truncation': 'disabled', + 'usage': { + 'input_tokens': 312, + 'input_tokens_details': { + 'cache_write_tokens': 0, + 'cached_tokens': 0, + }, + 'output_tokens': 144, + 'output_tokens_details': { + 'reasoning_tokens': 79, + }, + 'total_tokens': 456, + }, + 'user': None, + 'metadata': {}, +} + +GPT_6_1_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "openai/gpt-6.1-terra"}, + expected_provider_endpoint="/responses", + expected_provider_headers=GPT_5_6_TERRA_TEST_CASE.expected_provider_headers, + expected_provider_request={ + **OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS, + "model": "gpt-6.1-terra", + }, + mock_provider_response=GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE, + expected_litellm_response={ + **GPT_5_6_TERRA_TEST_CASE.expected_litellm_response, + "id": "resp_0b85038a4a37efb6006ae75c0cf84c87d1ac49e75264e5701f", + "model": "openai/gpt-6.1-terra", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I’ll read `shop/validation.py` first to check how it handles zero, negative, and non-integer quantities.", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": { + "arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}', + "name": "read_file", + }, + "id": "call_AJa3dZ4npr0KPN0SEwwl90xc", + "type": "function", + } + ], + "reasoning_items": [ + { + "type": "reasoning", + "id": "rs_0b85038a4a37efb6006ae75c0de1ac87d1a40d2ce0476272d9", + "encrypted_content": GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][0][ + "encrypted_content" + ], + "summary": [], + } + ], + "provider_specific_fields": { + "phase": "commentary", + "openai_responses_replay": ANY, + }, + }, + } + ], + "usage": { + "completion_tokens": 144, + "prompt_tokens": 312, + "total_tokens": 456, + "completion_tokens_details": {"reasoning_tokens": 79}, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + }, + "access_programs": {"cyber": "standard"}, + }, +) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py index b6a60721b65..11b14249af7 100644 --- a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py @@ -7,6 +7,7 @@ from integration.translation.chat_completions.bases.openai import ( GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, GPT_5_6_TERRA_TEST_CASE, + GPT_6_1_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -24,6 +25,7 @@ from integration.translation.runner import assert_translation GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_5_6_TERRA_TEST_CASE, + GPT_6_1_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/chat_completions/cross_model/README.md b/tests/integration/translation/chat_completions/cross_model/README.md index 99a4225480e..c14e3692549 100644 --- a/tests/integration/translation/chat_completions/cross_model/README.md +++ b/tests/integration/translation/chat_completions/cross_model/README.md @@ -5,10 +5,10 @@ Every row sends `CODING_AGENT_REQUEST` to `/v1/chat/completions`, changing only | Tier | Model versions | | --- | --- | | Small | Haiku 4.5 and 5.5, GPT 5 Mini and 5.4 Mini, GPT 5.6 Luna and 6 Luna | -| Mid | Opus 5 and 5.5, Sonnet 5.5, Gemini 3.8 Flash and 3.1 Pro Preview, GPT 5.6 Sol, 6 Sol and 6.1 Sol, GPT 5.6 Terra | +| Mid | Opus 5 and 5.5, Sonnet 5.5, Gemini 3.8 Flash and 3.1 Pro Preview, GPT 5.6 Sol, 6 Sol and 6.1 Sol, GPT 5.6 and 6.1 Terra | | High | Fable 5 and 5.1, Gemini 4 Argon, GPT 6 Astra | -The 52 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented +The 53 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented The request contains system instructions, conversation history, previous assistant tool calls and results, four function definitions, automatic parallel tools, low reasoning effort and an output limit. Each row exercises a tool-call response, raw streaming, SDK stream assembly and a follow-up that replays the returned assistant message and tool-call IDs @@ -33,7 +33,7 @@ Streaming cases store `expected_litellm_message` and `expected_litellm_usage`. M Usage requires nonnegative integer counts, positive prompt and completion counts, and `total_tokens == prompt_tokens + completion_tokens`. The complete normalized usage must equal the independently expected counters, including cached and reasoning tokens and derived text/cache details. Missing, changed or unexpected counters fail. Only null optional fields, the explicitly listed provider metadata/duplicate counters and computed `cost` are excluded. Thus a wrong prompt count with an adjusted total still fails -The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 52 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips +The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 53 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips ## Recorded upstreams diff --git a/tests/integration/translation/chat_completions/cross_model/case.py b/tests/integration/translation/chat_completions/cross_model/case.py index 8dee0fae2ab..5b0b87b4079 100644 --- a/tests/integration/translation/chat_completions/cross_model/case.py +++ b/tests/integration/translation/chat_completions/cross_model/case.py @@ -182,6 +182,11 @@ MID: Final[tuple[AgentCases, ...]] = ( openai.GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE, openai.GPT_5_6_TERRA_AGENT_STREAMING_TEST_CASE, ), + ( + openai.GPT_6_1_TERRA_AGENT_TEST_CASE, + openai.GPT_6_1_TERRA_AGENT_FOLLOWUP_TEST_CASE, + openai.GPT_6_1_TERRA_AGENT_STREAMING_TEST_CASE, + ), (openai.GPT_6_SOL_TEST_CASE, openai.GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE, openai.GPT_6_SOL_AGENT_STREAMING_TEST_CASE), ( openai.GPT_6_1_SOL_AGENT_TEST_CASE, diff --git a/tests/integration/translation/chat_completions/cross_model/openai.py b/tests/integration/translation/chat_completions/cross_model/openai.py index 46e9277b7cb..e993aafaf36 100644 --- a/tests/integration/translation/chat_completions/cross_model/openai.py +++ b/tests/integration/translation/chat_completions/cross_model/openai.py @@ -2704,6 +2704,242 @@ GPT_5_6_TERRA_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( ) +def _gpt_6_1_terra_stream_value(value: JsonValue) -> JsonValue: + if isinstance(value, dict): + return { + key: {"cyber": "standard"} + if key == "access_programs" + else 1793547872 + if key == "created_at" + else 1793547874 + if key == "completed_at" and item is not None + else _gpt_6_1_terra_stream_value(item) + for key, item in value.items() + } + if isinstance(value, list): + return [_gpt_6_1_terra_stream_value(item) for item in value] + if isinstance(value, str): + return { + "resp_07964387f422fad6006ad04aa9820c87d0b00c0f7e8179e669": + "resp_067882b6e08b211f006ae75e608f7c87d1952b1695ca966b80", + "fc_07964387f422fad6006ad04aaa99d087d09bf5301d4def0fb9": + "fc_067882b6e08b211f006ae75e6284ac87d1854926589ccdeef8", + "fc_07964387f422fad6006ad04aaa99e487d0adda87c8afeac85a": + "fc_067882b6e08b211f006ae75e62850487d1b7b53aac9602ff43", + "call_xDdtGVzn464uEE7X8AuVRiHJ": "call_g1sYS5VaOpk7yFfZWb6C6FhG", + "call_yPhpkZWP5B2zJo4SPhCEAxW4": "call_qq6AhKCnaWY7iQsRGkj9tshL", + "gpt-5.6-terra": "gpt-6.1-terra", + "IFA": "4N0", + "RH": "wV", + }.get(value, value) + return value + + +GPT_6_1_TERRA_AGENT_TEST_CASE: Final = replace( + GPT_5_6_TERRA_AGENT_TEST_CASE, + litellm_request={**CODING_AGENT_REQUEST, "model": "openai/gpt-6.1-terra"}, + expected_provider_request={**OPENAI_RESPONSES_AGENT_REQUEST, "model": "gpt-6.1-terra"}, + mock_provider_response={ + **GPT_5_6_TERRA_AGENT_TEST_CASE.mock_provider_response, + "id": "resp_04952ef9a70cae0b006ae75e5e7ae487d19a9876588a88de0c", + "created_at": 1793547870, + "completed_at": 1793547872, + "model": "gpt-6.1-terra", + "access_programs": {"cyber": "standard"}, + "output": [ + { + "id": "fc_04952ef9a70cae0b006ae75e5fd56087d19886a8f152af5d41", + "type": "function_call", + "status": "completed", + "arguments": '{"query":"retries_remaining"}', + "call_id": "call_EtA7LwkZhItDqLqZtTbMuAMT", + "name": "search_code", + }, + { + "id": "fc_04952ef9a70cae0b006ae75e5fd57487d1871209cffbada8fb", + "type": "function_call", + "status": "completed", + "arguments": '{"path":"tests/test_retry.py"}', + "call_id": "call_lVIvxbSII29GTSNa284H8WhK", + "name": "run_tests", + }, + ], + }, + expected_litellm_response={ + **GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response, + "id": "resp_04952ef9a70cae0b006ae75e5e7ae487d19a9876588a88de0c", + "model": "openai/gpt-6.1-terra", + "choices": [ + { + **GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response["choices"][0], + "message": { + **GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response["choices"][0]["message"], + "tool_calls": [ + { + "index": 0, + "function": {"arguments": '{"query":"retries_remaining"}', "name": "search_code"}, + "id": "call_EtA7LwkZhItDqLqZtTbMuAMT", + "type": "function", + }, + { + "index": 1, + "function": {"arguments": '{"path":"tests/test_retry.py"}', "name": "run_tests"}, + "id": "call_lVIvxbSII29GTSNa284H8WhK", + "type": "function", + }, + ], + }, + } + ], + "access_programs": {"cyber": "standard"}, + }, +) + +GPT_6_1_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( + GPT_6_1_TERRA_AGENT_TEST_CASE, + scenario="agent_followup", + litellm_request=followup_request( + GPT_6_1_TERRA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_6_1_TERRA_AGENT_TEST_CASE.expected_litellm_response), + ), + expected_provider_request={ + **GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_provider_request, + "model": "gpt-6.1-terra", + "input": [ + *list_value(OPENAI_RESPONSES_AGENT_REQUEST["input"]), + { + "type": "function_call", + "call_id": "call_EtA7LwkZhItDqLqZtTbMuAMT", + "name": "search_code", + "arguments": '{"query":"retries_remaining"}', + }, + { + "type": "function_call", + "call_id": "call_lVIvxbSII29GTSNa284H8WhK", + "name": "run_tests", + "arguments": '{"path":"tests/test_retry.py"}', + }, + { + "type": "function_call_output", + "call_id": "call_EtA7LwkZhItDqLqZtTbMuAMT", + "output": [ + { + "type": "input_text", + "text": '{"matches": [{"path": "src/worker.py", "line": 12, "text": "remaining = ' + 'retries_remaining(3, attempts)"}]}', + } + ], + }, + { + "type": "function_call_output", + "call_id": "call_lVIvxbSII29GTSNa284H8WhK", + "output": [ + { + "type": "input_text", + "text": '{"exit_code": 1, "output": "FAILED tests/test_retry.py::test_retries_remaining - ' + 'assert 4 == 2"}', + } + ], + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Stop after these diagnostics. Do not call more tools or change files. Summarize the " + "bug, the caller found and the failed test in two short sentences.", + } + ], + }, + ], + }, + mock_provider_response={ + **GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.mock_provider_response, + "id": "resp_0b23ebd19ad1a426006ae75e8c1de887d1a118f6e1dd5930de", + "created_at": 1793547916, + "completed_at": 1793547917, + "model": "gpt-6.1-terra", + "access_programs": {"cyber": "standard"}, + "output": [ + { + "id": "msg_0b23ebd19ad1a426006ae75e8caa3087d1b36f6b0c2fa883dd", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "`retries_remaining` adds attempts to the limit instead of subtracting them and clamping the result " + "at zero; a caller was found in `src/worker.py`. The test failed because " + "`retries_remaining(3, 1)` returned 4 instead of 2.", + } + ], + "phase": "final_answer", + "role": "assistant", + } + ], + "usage": { + "input_tokens": 619, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 60, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 679, + }, + }, + expected_litellm_response={ + **GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response, + "id": "resp_0b23ebd19ad1a426006ae75e8c1de887d1a118f6e1dd5930de", + "model": "openai/gpt-6.1-terra", + "choices": [ + { + **GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response["choices"][0], + "message": { + **GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response["choices"][0]["message"], + "content": "`retries_remaining` adds attempts to the limit instead of subtracting them and clamping " + "the result at zero; a caller was found in `src/worker.py`. The test failed because " + "`retries_remaining(3, 1)` returned 4 instead of 2.", + }, + } + ], + "usage": { + **GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response["usage"], + "completion_tokens": 60, + "prompt_tokens": 619, + "total_tokens": 679, + }, + "access_programs": {"cyber": "standard"}, + }, +) + +GPT_6_1_TERRA_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( + scenario="agent_streaming", + litellm_endpoint=GPT_6_1_TERRA_AGENT_TEST_CASE.litellm_endpoint, + litellm_request={ + **GPT_6_1_TERRA_AGENT_TEST_CASE.litellm_request, + "stream": True, + "stream_options": {"include_usage": True}, + }, + expected_provider_headers=GPT_6_1_TERRA_AGENT_TEST_CASE.expected_provider_headers, + expected_provider_endpoint="/responses", + expected_provider_request={**OPENAI_RESPONSES_AGENT_REQUEST, "model": "gpt-6.1-terra", "stream": True}, + mock_provider_frames=tuple( + SSEFrame(data=_gpt_6_1_terra_stream_value(frame.data), event=frame.event) + for frame in GPT_5_6_TERRA_AGENT_STREAMING_TEST_CASE.mock_provider_frames + ), + expected_litellm_message=coding_agent_stream_message( + ("call_g1sYS5VaOpk7yFfZWb6C6FhG", "call_qq6AhKCnaWY7iQsRGkj9tshL"), + ), + expected_litellm_usage={ + "prompt_tokens": 448, + "completion_tokens": 53, + "total_tokens": 501, + "prompt_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0, "cache_creation_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }, +) + GPT_6_ASTRA_AGENT_TEST_CASE: Final = replace( GPT_6_ASTRA_TEST_CASE, scenario="agent_tools", diff --git a/tests/integration/translation/messages/bases/openai.py b/tests/integration/translation/messages/bases/openai.py index f53baeafec4..ffb4b14c7e5 100644 --- a/tests/integration/translation/messages/bases/openai.py +++ b/tests/integration/translation/messages/bases/openai.py @@ -1,4 +1,5 @@ from typing import Final +from unittest.mock import ANY from integration.translation.case import TranslationTestCase from integration.translation.messages.templates import MESSAGES_BRIDGE_RESPONSE, MESSAGES_REQUEST @@ -100,6 +101,7 @@ GPT_5_4_TEST_CASE: Final = TranslationTestCase( ) + GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -784,3 +786,243 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( ], }, ) + + +GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE: Final = { + 'id': 'resp_0a3b11558c0819df006ae75c101de487d1812988a3147f6964', + 'object': 'response', + 'created_at': 1793547280, + 'status': 'completed', + 'access_programs': { + 'cyber': 'standard', + }, + 'background': False, + 'billing': { + 'payer': 'developer', + }, + 'completed_at': 1793547282, + 'error': None, + 'frequency_penalty': 0.0, + 'incomplete_details': None, + 'instructions': ( + 'You help maintain a synthetic Python project. Read the relevant files before suggesting ' + 'a fix. Explain which file you need next, then call a read-only tool. Never change files.' + ' Use the messages client for this project review. Consider zero, negative and non-intege' + 'r quantities before choosing the next file to inspect.' + ), + 'max_output_tokens': 1024, + 'max_tool_calls': None, + 'model': 'gpt-6.1-terra', + 'moderation': None, + 'output': [ + { + 'id': 'rs_0a3b11558c0819df006ae75c10fde087d192db4aa5ea7cf7be', + 'type': 'reasoning', + 'content': [], + 'encrypted_content': ( + 'gAAAAABq51wSSVCQB1815HYz5pLz1k32-dUjO-X-4b7LQG23OQZx_gqLXLuuL4xBCieTrvYatU55pFIxiQFXgtqG' + '6U1VtkToUospW-zTvH4roGicTQdWAovuoGt_w7rIFKNd0DUt8XB82WtVnogSd-ZQi4yJ3fjJLrR8nGTG596tQoGX' + '5TIFWeprIbpd99K1-bmu7dupFVg55x8zWfUx2e7wyYRFJ4JZLdIy1_2AwPaf5AjiZn5hnYXdGL3p7X_Bv_nkoRa4' + '1EE592bGS0K_GAxxVxH2sAdzQIOQf_G_E4fvnHRegGK61AgZQ9PVn2dXhGCXa-O0j1aYEx5pECMSfHaqExkx5-Nk' + 'z3l6Y7RmkoUKduquneXcZSRX-COeDTcts0hRFMTnlNQ5obc3tY_IEST-JDpKdsjL-6YEHhtJNrdED_DIj0vVrQgg' + '_3g0LfnDGSqtunsHs-KU4He_jYBJwXHNB2xbe5DMVAyNcqm7TOKRDejB_ym6cA_8j-oTXIumSnAao6I-l3WYU2yk' + 'tprvhby5FPnPhspqCYVu6nn5LxO7wyQOm_bOK4LcCojyp5OGRgCjtZ7AhAytPSryHt_gSMeEmQ11REFYh3PTrPt5' + 'ZqYYHW-miQ4Kgg80tn2I66EydrVEEaqZp8sQ7XgbFr7CG_6DOVGIGB819hLpA6clAZxNuUc8bVTwvT_o-wOLOePr' + 'Fa28kMGuucFWgLKW3fxQBh5MVHfMca-T8shc9aK8JLq7H74xfBaNWdXyNpMlNYr_kPC8LHqh2mKXMyG_17yShf-f' + 'h4MIY7dHrHr15X-aw8tAFBUZAkTTQF9Da4Cn09oX-yGGs1vW75HAPAdaBqVvAu_MAHXn_C3ACA4mKe7N4ihQWcVD' + 'zw9e0K6tKWwOYybCLenWnlJ64ooPajGtQvdUfYx4SGmcxTPd18q1_djz-J5k7T8gdqwiD5MEdJU3Trsrw7eULiFE' + 'qJu9oYrgsiVO-lsjIub5MPLCfKb8_L703ls2WBQ64BT-MZw2qgjKdHXk3f9KjRFaWM_HIJgaMrG6x9H2KE5Sqku9' + 'o1dANEH0ZpxyBb4lIofWnTrrXALN763TCQe0BC0IwmaoJBfAFThzhLbCuQBLsNZj4vi8rC7W6xSBxQ1UYCgSrZPh' + 'm8JUoGs4RWYgq7-mD3sB2jcRedcoG9RqxpF0h7_DIzHqk3UVnSpNiRnh_wGp4x_-lO2Ms6qp48QpSYOMJmG_KcRI' + '6qhcxTg2B3tU4965f2Iz0ghSzRiSctki1CPvWcRBtLv6LqLLoBK0NTgoByR66rUB7s6yYe6coP_OSkR0GFUCqH_X' + 'w-kcd-VckGJxKUm5tkejTjjrEXq3LiClTn9NUUgmtaWPEmZoHdNElyu1xiIK-0CTCb07zN_EK2_Nls5SRsWZBRFt' + '3YCGcw9vK2jSyx28cRwdBmb1fWXAQ3FYAl1uISH5kdcajQvWv1rhHPMW828OYZTZxG1whGwNPxOTWwj2sHuxvMwG' + 'MmOnhdTnIX-5qoNafrWraIv8jBC4Y4Z_hlCaB89Ba4Y_-B5-0ClNLGOx6q9fzj3wY4H3RAymTbi9pqqUpuX7wWBG' + 'uL9KggI6a5A09tmTIpucIsHY7dH6c12Xrls7vkrxlQh_7iix67VUj9UmspFvnCSVmX0W3IdPszNZpVHNpm_KOD-v' + 'Y5fSNhkTk_hTjjlSMhSe6Ki7BarnMzPo8j7EHVZcgmoAOMoNDHvIEPjbCSXphGr9qYfQ4ajffaloJLh-51e0C8mJ' + '1daJ2xs7ZT7uJI0syeeKVRnqxDYlvoAEKl1KrUrX9SKceoVANOla-DMWs3TSsUL6-ccPqzJFDdPn6aVoIxS4IMGa' + 'nn5oWXVRxLUS01d19mVPn_XoNXrLwi2GCW0r8GfowHyeEgW535xwNt5uEGmmmV17baclC6x1GUo3xPkKpnPaVgf7' + 'jIjBP7P8-FOcHoIFB9MfUL_Ot2UdK26tc7vL7CgUuch08WPZOklx8Vsv7N0OIzWiczrA09il52Z_QyyTB5VmfiYS' + 'X6cLsnFCTh7fTSDs4p80vEfBWqKCW3M0isAWLTEaQ7kwpmHulUI_fvxJg2Bn-lV48aKaLeZT0FAS_WRP-cGBIb79' + 's8TlmnRcyBJhR0LbMEyMHqo4Y7hwJAyTZkBtejuCyyE3b-2QKq2w-GtcgD5MI9b6EOt7ET6EbFFS' + ), + 'summary': [], + }, + { + 'id': 'msg_0a3b11558c0819df006ae75c11a3c087d18ee211f7533e565f', + 'type': 'message', + 'status': 'completed', + 'content': [ + { + 'type': 'output_text', + 'annotations': [], + 'logprobs': [], + 'text': ( + 'I’ll read `shop/validation.py` first, checking how it handles zero, negative, and non-in' + 'teger quantities.' + ), + }, + ], + 'phase': 'commentary', + 'role': 'assistant', + }, + { + 'id': 'fc_0a3b11558c0819df006ae75c12142887d18b71a86e0c02ae8c', + 'type': 'function_call', + 'status': 'completed', + 'arguments': '{"path":"shop/validation.py","start_line":1,"end_line":150}', + 'call_id': 'call_Fnk0hANehtgpqZaZieVoZUYK', + 'name': 'read_file', + }, + ], + 'parallel_tool_calls': True, + 'presence_penalty': 0.0, + 'previous_response_id': None, + 'prompt_cache_key': None, + 'prompt_cache_retention': '24h', + 'reasoning': { + 'context': 'all_turns', + 'effort': 'high', + 'mode': 'standard', + 'summary': None, + }, + 'safety_identifier': None, + 'service_tier': 'default', + 'store': True, + 'temperature': 1.0, + 'text': { + 'format': { + 'type': 'text', + }, + 'verbosity': 'medium', + }, + 'tool_choice': 'auto', + 'tool_usage': { + 'image_gen': { + 'input_tokens': 0, + 'input_tokens_details': { + 'image_tokens': 0, + 'text_tokens': 0, + }, + 'output_tokens': 0, + 'output_tokens_details': { + 'image_tokens': 0, + 'text_tokens': 0, + }, + 'total_tokens': 0, + }, + 'web_search': { + 'num_requests': 0, + }, + }, + 'tools': [ + { + 'type': 'function', + 'description': 'Search source and test files for a literal term.', + 'name': 'search_files', + 'output_schema': None, + 'parameters': { + 'type': 'object', + 'properties': { + 'query': { + 'type': 'string', + }, + 'paths': { + 'type': 'array', + 'items': { + 'type': 'string', + }, + }, + }, + 'required': [ + 'query', + 'paths', + ], + 'additionalProperties': False, + }, + 'strict': False, + }, + { + 'type': 'function', + 'description': 'Read a range of lines from a project file.', + 'name': 'read_file', + 'output_schema': None, + 'parameters': { + 'type': 'object', + 'properties': { + 'path': { + 'type': 'string', + }, + 'start_line': { + 'type': 'integer', + 'minimum': 1, + }, + 'end_line': { + 'type': 'integer', + 'minimum': 1, + }, + }, + 'required': [ + 'path', + 'start_line', + 'end_line', + ], + 'additionalProperties': False, + }, + 'strict': False, + }, + ], + 'top_logprobs': 0, + 'top_p': 0.98, + 'truncation': 'disabled', + 'usage': { + 'input_tokens': 309, + 'input_tokens_details': { + 'cache_write_tokens': 0, + 'cached_tokens': 0, + }, + 'output_tokens': 121, + 'output_tokens_details': { + 'reasoning_tokens': 56, + }, + 'total_tokens': 430, + }, + 'user': None, + 'metadata': {}, +} + +GPT_6_1_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={**MESSAGES_REQUEST, "model": "openai/gpt-6.1-terra"}, + expected_provider_endpoint="/responses", + expected_provider_headers=GPT_5_6_TERRA_TEST_CASE.expected_provider_headers, + expected_provider_request={ + **OPENAI_RESPONSES_REQUEST_FROM_MESSAGES, + "model": "gpt-6.1-terra", + }, + mock_provider_response=GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE, + expected_litellm_response={ + **GPT_5_6_TERRA_TEST_CASE.expected_litellm_response, + "id": ANY, + "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + + GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][0]["encrypted_content"], + }, + { + "type": "text", + "text": "I’ll read `shop/validation.py` first, checking how it handles zero, negative, and non-integer quantities.", + }, + { + "type": "tool_use", + "id": "call_Fnk0hANehtgpqZaZieVoZUYK", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 150}, + }, + ], + "usage": {"input_tokens": 309, "output_tokens": 121}, + "model": "openai/gpt-6.1-terra", + }, +) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_openai.py b/tests/integration/translation/messages/basic/test_messages_basic_openai.py index 38d864411c0..fc9b8b25a56 100644 --- a/tests/integration/translation/messages/basic/test_messages_basic_openai.py +++ b/tests/integration/translation/messages/basic/test_messages_basic_openai.py @@ -7,6 +7,7 @@ from integration.translation.messages.bases.openai import ( GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, GPT_5_6_TERRA_TEST_CASE, + GPT_6_1_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -24,6 +25,7 @@ from integration.translation.runner import assert_translation GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_5_6_TERRA_TEST_CASE, + GPT_6_1_TERRA_TEST_CASE, ], ids=lambda case: case.id, ) diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py index 35a0d5f087d..17f51b35468 100644 --- a/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_anthropic.py @@ -23,6 +23,78 @@ SIGNATURE_1: Final = ( "1FW7qRmOMOKJKhbheeMpsTs7XvdvsiDQqgM4PAJt4cwgGAE=" ) +CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_1: Final = ( + "CAQS2gkKEAgTGAI4AUIIdGhpbmtpbmcSDCKQrdbAsNSlkOf3zBoMeduhYaXR7h2XbdAKIjAtYK1x82HeQuy3OMDRyk8Ru1NL6Bo2p4jMGpeqJSUAqMBl" + "hxlFnMpq2arnT1bWs3kq9wjg19gHCDS58DA+DUr3YdYx6gy1aX/Qn1XeFVB1L/kSAHlK0Wuxzn41ZM3D3xdGPeg9El2AIQryUlRs/POxJdqRzXes2H5U" + "fUU1f/azCoeDJv9RRk234jIMkgBCWiLstzExNwxL+kvTlY2mrQXQ+U9mhXRJC90AilZlFIY8fuRhL8HMFA6d0rqMisr3wLd9M7ACwVP3cOZXx1Wa9Sfb" + "2yiutDHkG4fI61xE8cDTYvOqtyY6ZQ9juiV6/rOPDSP+85sLVVmH+wJUJXIEfYBRCmytchPqR/Y/Q1EdMnmiLYfssjWsj8NkfsEBjOHdlIYDTzvplkEK" + "y2SHSBwQeHcahHr1Y1napFobIRjVuQnPOMGvhNLUJZwN4zJLAQXzxO5hK2KrFOg66nX3A/+77CGVUr+CpC+/Wbkyc7LQNfI+xzsdRjC7epieVPbgVFDS" + "Rt0a7nppVA7JbYzZh04UdCoD7z2dDvX3lhE3WDdyICUatVgsRaU0w1Qmiu9fOwrfU6I29PaXDGw1i4pth6uL8GAS/ZuUC6xFhggEoXs497MgrD3N0uLQ" + "69x+IGmuRzIPbcgWjC3IlNMI2se/FpfWXJAsScnCVWRqfBL8crjsmdbuwy4L/usMNT87H3vngSbpd9dZuKpKBW18Y+OPLQiF2im/Pv+zZpA1RJkOvXPr" + "ru8eiHu9z2XKVIJgdxKkRjLgi4/JVnN54q3bdkSz7KARIiRc4EzKsyU3/PzJJN1Fev10hxbU4V+Pkv6hLcjzIL9pMGAYjZXrjpxyw+PTVt7jK2k1C1SB" + "BXpHdmntJJ30E94IpSISyNIW9Twi3KPYcYhj/rd9WA5UwvhDtbvril6ET10JlqzV36l8RxWUPhlXegJgtEMdnjNplhfDeGsMwg8e3OL1GNRUQoqQNd1Q" + "jP3bpHx0yaxaRlkGwgtIGXj2bxuwLwn5otMEMGzQEnnQVNHdjAIpuHunLH9k01S86ir6s5cP05+U1Y1V1U4dZiFVnr4vSwWLIZLpRg2k/O4/inqzIjOY" + "LD9xE+d+Mm55vKnbYtIv+DM1GHpKdvbzSfa1czjV6YT1PKotUZpg4ul+uyQqZYhpq9dGYUttevBz5tPIs+XUL6nx3Kj8WnPtkaAD+MuH0+yuALNdwzGb" + "4sxxAu4sYsbxHCzpriX+JLjgUq2bO2Y7ve1nHVsh1B1bzyatgv9hnAQyO7uLMP0IICF3F43zJtgol3/gng5bf1T6EeFHDyZAUxGAqbWGjPwh5awEfdLe" + "SYc2AtK84ieDLU0LfhvuGRdmAcVXrGmF1D/mhHg95Hs8fKt5QN7O1/gUSsC0TvXGrpbYeE5OH/O2aFFzimURUyUp7W8f6UFNNy4ZHIjdUqomnUL8XH+o" + "Rf69GBqbv8BsfPQuq9D6nAvFk+suLYhLvWdFRy/sMA/gXh2mq21t2bFXQXOjgq/dQ7GUJx0vU86alotjdPovkksZAqzHi1s5h3WrQ1h3KxF4pfVO309h" + "lQiFvg3fNPaDARqFFci+7ToS0kQXk4r95A6XzGMYAQ==" +) + +CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_2: Final = ( + "CAQS1gcKEQgTGAI4AUIJbmFycmF0aW9uEgxdyD7sRZxxSBB6HUEaDB0dBrarQ/maHvvYRiIwNo20xJqSuL5tpy0fFBXRggkfQd3MO+ehcdkzkAP1bNQF" + "I7P1/rfhUbkzeiZI7NDDKvIG0xpIrPlf76ufYrZ2d9+04GJUImemCjrKGxjniCVaBpqVqJp0K7qWu24HZqZjX/pcKUi//W0t73h1BFmkUEy/8XSvsuUs" + "u7lJqmMXACCXhkXZhg/EnswdEzKhYwVDay6cGqfGUUo+NBTmhMjYjy69h7x1+RPuEWTNNMhusS2K9U1hFwZjqEdjo/4CsWZzZC6/9WC213voHd6ewjly" + "QQLjEKJgoQ4Mnlyep7NVe3b4Hzu0pChITa/PUyhEGEoQhIZoMx3/hYAf3HwqveaPBQRmrjf8m3bvcbh7R0hTJHS3dpUrz5RE6d8zSy6EGWTDTYqb6JpQ" + "DdL0+5UxQMZEc6JTcpcBpfxJvS2KFyxyZwOkt02OdjEoTkPxaQTK9fnjVSq9xp5qLm/RC03XnBdKBG5Elk/Dxj4UqPphkKN+UQCyYKivCpkU/N0/MVm1" + "/Rp0x5OQuZ/xVSbdyuF8ijlmK/FZuZ8P3sR0v5gvfPGvDoijdWaljc/+ZiJ7CK9LtgoSk16IYrCbKtbL75S7yGTHnWgnHSUzuXorpkf5xGuGKucloLTf" + "DPaBsxw7mWUEYcoUE1LxUqip9j2p6Zf6BkOHX5TPgXB14F6194/vwIfoNV7QK/uXFT2kZ/6m4YDfTzmvDCtUP9PcfbyjOIPHXukL/KzbaunT91spQf28" + "7CfAbZp0Tl0ATcVut63UB/3e26zxhZ45ZOLyjahllcLY43FWmPLM+tTUPtNmKQ99+JVBS8A4rqKZwnovgFJG4wmjbeBClY691hNoNVsIfPQs0RM0zgIh" + "F4rutOqnPIBbcS/5eFGucSB3sAplbqDTQtHVIagaTFAgLtEBPxGqb9jze+Wwwt2RQ9xXE36bEEsgL+jgrd7CfiTs78RCV/G6tKzyfO5TJjQoX/8XUIag" + "C5zdRpNg7cTRcDIZPkTl0Jii/Pgrhd3xkBG5heg9YOnsm1xuHkmTjGGGx1sDMeBfLiRPNyYTDHVAqejbqB2dyGLvfsx3DZgSLnv4IBXcesQ0am673hqL" + "WYYMT7md2Z9JKoYGnTXqkbmXDkuf6H+RasWhGYkNJtF+XBxUwHVvbMjQaO2tJpQhv8msR3yORQn2Lnt5OF2ws69Irqt5tdkQCyCGn+O6xxOftE3gmEMZ" + "0Udpgod2UvJj1Vz2udQhXAkmNRmPgLmp6jRWU7bmGAE=" +) + +CLAUDE_SONNET_5_5_THINKING_BUDGET_CONTENT: Final = [ + { + "type": "thinking", + "thinking": "", + "signature": CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_1, + }, + { + "type": "thinking", + "thinking": "", + "signature": CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_2, + }, + { + "type": "tool_use", + "id": "toolu_01CndWUrVgzs4gR9gg1R6w44", + "name": "read_file", + "input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_014RmKHkVks5GvQY6qZoXboC", + "name": "read_file", + "input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40}, + "caller": {"type": "direct"}, + }, +] + +CLAUDE_SONNET_5_5_THINKING_BUDGET_USAGE: Final = { + "input_tokens": 4, + "cache_creation_input_tokens": 873, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 449, + "output_tokens_details": {"thinking_tokens": 253}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", +} + CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace( CLAUDE_SONNET_4_6_TEST_CASE, scenario="thinking_budget", @@ -74,9 +146,44 @@ CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace( }, ) +CLAUDE_SONNET_5_5_THINKING_BUDGET_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_TEST_CASE, + scenario="thinking_budget", + litellm_request={ + **CLAUDE_SONNET_5_5_TEST_CASE.litellm_request, + "max_tokens": 2048, + "thinking": {"type": "enabled", "budget_tokens": 1024}, + }, + expected_provider_request={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request, + "max_tokens": 2048, + "thinking": {"type": "adaptive"}, + }, + mock_provider_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response, + "id": "msg_011CgcUJqytdPvGPzVpJNKEQ", + "content": CLAUDE_SONNET_5_5_THINKING_BUDGET_CONTENT, + "usage": CLAUDE_SONNET_5_5_THINKING_BUDGET_USAGE, + }, + expected_litellm_response={ + **CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response, + "id": "msg_011CgcUJqytdPvGPzVpJNKEQ", + "content": CLAUDE_SONNET_5_5_THINKING_BUDGET_CONTENT, + "usage": CLAUDE_SONNET_5_5_THINKING_BUDGET_USAGE, + }, +) + @pytest.mark.parametrize( - "case", [CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE], ids=(CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE.id,) + "case", + [ + CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE, + CLAUDE_SONNET_5_5_THINKING_BUDGET_TEST_CASE, + ], + ids=( + CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE.id, + CLAUDE_SONNET_5_5_THINKING_BUDGET_TEST_CASE.id, + ), ) def test_messages_reasoning_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/responses/bases/openai.py b/tests/integration/translation/responses/bases/openai.py index 8e379e1019f..7a6515bc2d3 100644 --- a/tests/integration/translation/responses/bases/openai.py +++ b/tests/integration/translation/responses/bases/openai.py @@ -161,6 +161,7 @@ GPT_5_4_TEST_CASE: Final = TranslationTestCase( ) + GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/responses", @@ -1158,3 +1159,256 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( "top_logprobs": 0, }, ) + + +GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE: Final = { + 'id': 'resp_0489dbceb3070b9a006ae75c12ef5487d1a7cb64a92d8774d1', + 'object': 'response', + 'created_at': 1793547282, + 'status': 'completed', + 'access_programs': { + 'cyber': 'standard', + }, + 'background': False, + 'billing': { + 'payer': 'developer', + }, + 'completed_at': 1793547285, + 'error': None, + 'frequency_penalty': 0.0, + 'incomplete_details': None, + 'instructions': ( + 'You help maintain a synthetic Python project. Read the relevant files before suggesting ' + 'a fix. Explain which file you need next, then call a read-only tool. Never change files.' + ' Use the responses client for this project review. Consider zero, negative and non-integ' + 'er quantities before choosing the next file to inspect.' + ), + 'max_output_tokens': 1024, + 'max_tool_calls': None, + 'model': 'gpt-6.1-terra', + 'moderation': None, + 'output': [ + { + 'id': 'rs_0489dbceb3070b9a006ae75c13f80487d1ab5a5153f0461603', + 'type': 'reasoning', + 'content': [], + 'encrypted_content': ( + 'gAAAAABq51wVlJWN-LTJqqhWvFyWjY49-yFum5T56W8eITsd-qT0V290Gw80RoL9VVX4jvv6cfpCOb73lk6WHbMW' + 'q0tjO3yeVbczh_0YDEhG4-CGvQaElquYG1AK2fABB3X_J3xPglgIB9OnPuQiLoYBQHqYy3U-0TinuAr0lFCcWFex' + 'r1JAcM5JExGND9kJUCXx6Zfpt1_e4M467kCKmYW_pZ3Y0Ndf5uFLooX18Xf9-GBGiB6TtaS_atSGAVaeBYFYaNc4' + 'WIaTG-675Lnd6dHGQIrV4jBW7Xf4QKlQU1hFZCkUg0JW54skR8c7iMdA8qvdo2ieZ6JgnxU0gujk3OycBhXM1SaL' + 'P-MenI0QsOFcWASEozr-U53h-b4Gek9TBwY1PGMA8hITFw7tEwpt-s9LvZuDr0kcknR5d8RYyf1owYV-O668ykR5' + 'VUzheo2NOVUzQIe9xQorENDnxrdya8gO54HxDV9ShzbZ9e9ymv9iRH6jCjSaTOXJdCPjfxd5aYoPTnfibh0RYxM2' + 'UlcMZhpv5aXfu7c7hUbXFKx1eky48LAE0s1DjcVGa0YCqFNWAJEYJR7ZHpcaN8dzAc4HwNZZo2Oh15AfybRkm8Zh' + 'Jb5EalmOETrLK0_jxlH29Ln4xqpwlBHWeq6l0z_Qt3CqihV4dV5gsRGaJky6NoSn7UjpWaCzAkFjf1R4nIvqsJDM' + 'CmFAlS32ne_nL9spLd3uUAXzMCLwnfCs30xp8XxDwFwYq7LweRjdBkEBeOibTsaACR_b4Cp5GwICWsqrzXRpSh7K' + 'w73iewX7dq6gOiXZlf3bFg6A35h2EIVUPwBnEViN-mwgebe1pLH3vFUHocR3K70PIs7qLlWE5-ZsX_qVd_E1EmbR' + 'zom8leGZ798zodGpJyy-aTI49RgsPwk4gWugaoB8MKI3ykkmAeLbF3pDRvzTg3ZxF49fFS-0ALrQhB2177MZWymu' + 'ZHh_SQFi12KCMhzXxO8GkzeMDFV0OKezVkeveafa1eRfWReJJqaobBWd705vLaqidOTIYErHGVhEDKL9judPl0nl' + 'UhPXh4lDw7LN1qVXUiTFateiic7m-y-ibeTJj500tooNOJG4BSlmHYgZF1hA1FUEtQlDWXFbTTcz8yTXLB0ucbXb' + 'GudNKyI6u2oTPn0kaZvHuw5tra6_Lkp2G1DJ9ZL-tMLRLH8Fca4RzVTJnoqHZ2S-rl7ukKcYKQaar5ZE4I2zL_F7' + 'n_4Zr6lIlKOapXkcvso5UpThSy6OnMcUVh9DqWBKcCcQRA3TSshiQHNuY8xQIoNdm7YL4pRzufuvykbINnfUS0UD' + 'RUOSgNGImQHwNggEJysPx9ctqRXW0hpvcF5wrZbAtGLa6Eun-mBQ_z6-b7vT_uIJmzD9PWoaMfcHEb5fa4nXVFsx' + 'xBIvlMZ8cM7Yims2aG9PT1V7UrsUyVPP_K8fHJdheOM4oy4nC77fYrEjSUJeh-uCJ6wbXE0PuNBQWivWcwE9VV7U' + 'jXiIsCEblEVGHlhKL2EEajfB0tEYDOTVV4B2agwwLU5S6JX91UCpcH1dvFTYS_roXMDf3CWmo0JseRW2oZzxcCoP' + 'NZVvGEMhTCsitwVUEaRBPfNtLiplhbOrUx80IY38kRld9OQb6KxOvyuxqG4LT4_afTQImSGEZJQSW8qo9GzKN_sJ' + '9iQvGw6tgqrwqmmLKXjFDp3I5FB29xZAOoxhnapGIKL6KUuehubY_kM5ok21PvPuvfsY-gaFzFQB5dJsaBNZoDaJ' + '4cB1pw-tXmP4gUECk0PkoMQEekSzBJLfwk7jbIMS_YSzpE1RcqFenprMVkq-HBfAaaKIQxRi2qB3WM2VaoYGDDFG' + 'Xi3E3lc6x0D64-vXtxo4EPK-eXPC-ThPShl6POgKuoQZjDvo918hJDn3PCBq7jVXeEAgpOvLkSWQnen_khH5y4Un' + 'tF8L1auZfD2YugvETRdQsQKuX5ZCJ0_gT9L2Olc3QXbZoE4wzUBiyBdLlysDZanR4e3bgff3ZcsH02Et9iO2ljbO' + '8ELqlEzfZWV6lU_MT8GtHJfeIzDPp4UDraHdwyQYtkbJ9LNaub0JR4sushyYO6fLZLB45GqP8TJtCVlorF6okoy4' + 'AAqBvLb9vu2PAD8a9pjfZ6WNypYJelZ9eEsBTQ_sZGWnBFmvNhjZbFoXHkNAgp9HL0msO7ydRPnL_BPytOMXHiqy' + '5VcVaLnRCno3wm-QftrJyrp8AQKnjZL8DEXAYUUrAGofQ5wFA7HhjKGPTTLxd-cahYh9Lkkfwp3fUj3YgPTYDM35' + 'I4dpgUG9UCutIYwtmgdM-uKiot2w' + ), + 'summary': [], + }, + { + 'id': 'msg_0489dbceb3070b9a006ae75c14b9d487d183c51260c69dd557', + 'type': 'message', + 'status': 'completed', + 'content': [ + { + 'type': 'output_text', + 'annotations': [], + 'logprobs': [], + 'text': ( + 'I’ll read `shop/validation.py` first to check how it handles zero, negative, and non-int' + 'eger quantities.' + ), + }, + ], + 'phase': 'commentary', + 'role': 'assistant', + }, + { + 'id': 'fc_0489dbceb3070b9a006ae75c152c9887d19f48a7ea1647f1e4', + 'type': 'function_call', + 'status': 'completed', + 'arguments': '{"path":"shop/validation.py","start_line":1,"end_line":200}', + 'call_id': 'call_J39X3VwRp2iE5zJYYELJRTOn', + 'name': 'read_file', + }, + ], + 'parallel_tool_calls': True, + 'presence_penalty': 0.0, + 'previous_response_id': None, + 'prompt_cache_key': None, + 'prompt_cache_retention': '24h', + 'reasoning': { + 'context': 'all_turns', + 'effort': 'high', + 'mode': 'standard', + 'summary': None, + }, + 'safety_identifier': None, + 'service_tier': 'default', + 'store': True, + 'temperature': 1.0, + 'text': { + 'format': { + 'type': 'text', + }, + 'verbosity': 'medium', + }, + 'tool_choice': 'auto', + 'tool_usage': { + 'image_gen': { + 'input_tokens': 0, + 'input_tokens_details': { + 'image_tokens': 0, + 'text_tokens': 0, + }, + 'output_tokens': 0, + 'output_tokens_details': { + 'image_tokens': 0, + 'text_tokens': 0, + }, + 'total_tokens': 0, + }, + 'web_search': { + 'num_requests': 0, + }, + }, + 'tools': [ + { + 'type': 'function', + 'description': 'Search source and test files for a literal term.', + 'name': 'search_files', + 'output_schema': None, + 'parameters': { + 'type': 'object', + 'properties': { + 'query': { + 'type': 'string', + }, + 'paths': { + 'type': 'array', + 'items': { + 'type': 'string', + }, + }, + }, + 'required': [ + 'query', + 'paths', + ], + 'additionalProperties': False, + }, + 'strict': True, + }, + { + 'type': 'function', + 'description': 'Read a range of lines from a project file.', + 'name': 'read_file', + 'output_schema': None, + 'parameters': { + 'type': 'object', + 'properties': { + 'path': { + 'type': 'string', + }, + 'start_line': { + 'type': 'integer', + 'minimum': 1, + }, + 'end_line': { + 'type': 'integer', + 'minimum': 1, + }, + }, + 'required': [ + 'path', + 'start_line', + 'end_line', + ], + 'additionalProperties': False, + }, + 'strict': True, + }, + ], + 'top_logprobs': 0, + 'top_p': 0.98, + 'truncation': 'disabled', + 'usage': { + 'input_tokens': 309, + 'input_tokens_details': { + 'cache_write_tokens': 0, + 'cached_tokens': 0, + }, + 'output_tokens': 131, + 'output_tokens_details': { + 'reasoning_tokens': 66, + }, + 'total_tokens': 440, + }, + 'user': None, + 'metadata': {}, +} + +GPT_6_1_TERRA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={**RESPONSES_REQUEST, "model": "openai/gpt-6.1-terra"}, + expected_provider_endpoint="/responses", + expected_provider_headers=GPT_5_6_TERRA_TEST_CASE.expected_provider_headers, + expected_provider_request={ + **OPENAI_RESPONSES_REQUEST, + "model": "gpt-6.1-terra", + }, + mock_provider_response=GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE, + expected_litellm_response={ + **GPT_5_6_TERRA_TEST_CASE.expected_litellm_response, + "created_at": 1793547282, + "completed_at": 1793547285, + "model": "openai/gpt-6.1-terra", + "access_programs": {"cyber": "standard"}, + "output": [ + *GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][:2], + { + **GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][2], + "namespace": None, + }, + ], + "usage": { + "input_tokens": 309, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 131, + "output_tokens_details": { + "audio_tokens": None, + "reasoning_tokens": 66, + "text_tokens": None, + }, + "total_tokens": 440, + "cost": None, + }, + }, +) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_openai.py b/tests/integration/translation/responses/basic/test_responses_basic_openai.py index 0d5e46237d2..38b80e74eb2 100644 --- a/tests/integration/translation/responses/basic/test_responses_basic_openai.py +++ b/tests/integration/translation/responses/basic/test_responses_basic_openai.py @@ -7,6 +7,7 @@ from integration.translation.responses.bases.openai import ( GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, GPT_5_6_TERRA_TEST_CASE, + GPT_6_1_TERRA_TEST_CASE, GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_6_LUNA_TEST_CASE, @@ -24,6 +25,7 @@ from integration.translation.runner import assert_translation GPT_6_1_SOL_TEST_CASE, GPT_6_ASTRA_TEST_CASE, GPT_5_6_TERRA_TEST_CASE, + GPT_6_1_TERRA_TEST_CASE, ], ids=lambda case: case.id, )