mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge 1f32941dea into 527d882508
This commit is contained in:
commit
5123bbec56
11 changed files with 1133 additions and 4 deletions
|
|
@ -326,6 +326,11 @@ model_list:
|
|||
model: openai/gpt-5.6-terra
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-openai-key
|
||||
- model_name: openai/gpt-6.1-terra
|
||||
litellm_params:
|
||||
model: openai/gpt-6.1-terra
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-openai-key
|
||||
- model_name: openai/responses/gpt-5.6-sol
|
||||
litellm_params:
|
||||
model: openai/responses/gpt-5.6-sol
|
||||
|
|
|
|||
|
|
@ -451,6 +451,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase(
|
|||
)
|
||||
|
||||
|
||||
|
||||
GPT_5_MINI_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="agent_tools",
|
||||
litellm_endpoint="/v1/chat/completions",
|
||||
|
|
@ -1235,3 +1236,276 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase(
|
|||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE: Final = {
|
||||
'id': 'resp_0b85038a4a37efb6006ae75c0cf84c87d1ac49e75264e5701f',
|
||||
'object': 'response',
|
||||
'created_at': 1793547277,
|
||||
'status': 'completed',
|
||||
'access_programs': {
|
||||
'cyber': 'standard',
|
||||
},
|
||||
'background': False,
|
||||
'billing': {
|
||||
'payer': 'developer',
|
||||
},
|
||||
'completed_at': 1793547279,
|
||||
'error': None,
|
||||
'frequency_penalty': 0.0,
|
||||
'incomplete_details': None,
|
||||
'instructions': (
|
||||
'You help maintain a synthetic Python project. Read the relevant files before suggesting '
|
||||
'a fix. Explain which file you need next, then call a read-only tool. Never change files.'
|
||||
' Use the chat_completions client for this project review. Consider zero, negative and no'
|
||||
'n-integer quantities before choosing the next file to inspect.'
|
||||
),
|
||||
'max_output_tokens': 1024,
|
||||
'max_tool_calls': None,
|
||||
'model': 'gpt-6.1-terra',
|
||||
'moderation': None,
|
||||
'output': [
|
||||
{
|
||||
'id': 'rs_0b85038a4a37efb6006ae75c0de1ac87d1a40d2ce0476272d9',
|
||||
'type': 'reasoning',
|
||||
'content': [],
|
||||
'encrypted_content': (
|
||||
'gAAAAABq51wPWhH43zwDEF4VgA1Kgnfi13a_k7Ff26kP_ipp7PFCxc9W_3tlMScFQVruAcFY4-QJk3WYSiz_ErCS'
|
||||
'n7_u9vSRN1KB6ytyA4CMads8YIpqomBsURrtDZKtt26D2sp9DgthN5hg4sMsAiuOGkhNWwQ91cXOv6YXmKRQ3nTX'
|
||||
'ZhMXy07iuYELaE10_tszlQ6ZG0Etml2Ka59Y_Rod-NQHRtWSfowQrqhaYgwDimAMfuVdOo4myHtwIAWT8O4I2oAH'
|
||||
'BwXm2LcKPUh7DcUBNvdrqTSO9YfgFpmc82-y5xaHYcRd2nM0s5_AaQfISy1LANBI7l5yY-D_FxxISG6zyzOagbLO'
|
||||
'lLhdP3ftLpwAqu_xp13sJELtUR0r3itiDRxllGU9r0oIQ3b8bu3Z0bOH-1WWRBTEBUVCeDSadRIpa-gRJhZ-DSir'
|
||||
'wvZbqpZrKhn7LtaW0wgQlwuUZUCGk7MxdTrKe7Zw7K86iXY39outrZ4JACMUx5bez-_O0WVyuvjeIWAHKCSH9ofj'
|
||||
'bZXjPpG6mKaIggIGw8nZBr_uy60TVTvWvodJPoJ50lEDoi0ZClDgtQol7ypWSKBIVViuNMgHp27n4BFlGfl57Aef'
|
||||
'nD3PzqHFUNCawiH56wP1YgcsrwnlFCPKkFfgZHTGj190Rg8gTFRpOYfNLrI65AIR6LrUY091yqwAfifk3iyh5qy7'
|
||||
'YGtNyYw0IGTV3OYKoQJWCi3Pct6f9tNUenTtq7e-UT0OxVQxe_yyBPTm73TthwygwTSVczQ25q49Buzu7Dk7YWAw'
|
||||
'AvKGfvT9dqn7MsAV37PjBYTeRjZr23uNHIyPc8wHx5gq_1czCRDASUx6rEhQQzcvhG79ru2_apH61JWAYiVnfBAT'
|
||||
'zsvfabELZ9xdZxiY_p30L4AQKaV4MGPfYuqlKWpY3mxRUGjqres5KqVd5-rGWsfm4xlPYQpRnnKcNbEd1gNfDdc2'
|
||||
'IQUBmRB9zMVszhvWTFleLJrd84ztACnuC8DKxFSxihZ8QDKJbIEMqpYqm6ZUCljweUI--f9nYaoX9OIl5ch4tPoq'
|
||||
'7SVRdG1pxfAo7RubrgAYNiACntgbhpF8Wf9wWbXTxgBco4QvtdeEqEm6Fp--fK5uMvf2aUXL8mCda8Z-KZBWlO1v'
|
||||
'4xh8cHRJOkIU0rWNLYaPpmmEq109oL6B3gmSe7xrkAzhsD5ITBziXgZZvxY4r34JfLlBWRrx_1sEqFNTCtuaibVZ'
|
||||
'ZK15AysbTmMlumjfkP-tC8ygdtqJfSUUsVFzxgf8z6ThoP0MeMeuOKA79VeHpCnl758U80BIeUicDBBwwYIP8kpU'
|
||||
'ZyGqUTZcxDNQ80U9mXYMErrDf0PnzisTpw2Tm6gytBdjKfH07P272VfNJYHsO53l7oOkyUS1e9OdeWBEMaDje1B0'
|
||||
'QITl4-laHjbXnMIdZUdsxWTTiCljzzQQVoN8nQASBTvw-QJNFMqjAyBOt02bvCOVh7Jb9VJrTpXmVZKZ2HDHccma'
|
||||
'7EohOL0qklhQ6EXxPdn_Gs3-9pGESbfOFZaf01A6gH7iznMxRUMVYAUl4yqr8PFGpA-uEljFhScorVtN1Rm5QUGA'
|
||||
'qiCMUnVjeb40_swu8wlC5y7liGJXEiq0zo0wgrn_HU3FZXmWXbBHAkCWlFt8pz2CPIMNLUvc4HaN8Xlcq6kK2aX5'
|
||||
'OH_Q8tVUsIS_0SpoAjWVc-6V9a7veqoik2yQyujvjw0b9sESo9qyZeiYj0M73v6SqBfjW2ACTZk5v9V5xLYgjOxZ'
|
||||
'Pe-XALR50X3B0FEalWUkUC-uv0scJcHYxb8k2n21OPr4F0Lb01gdTFyk9cbfXyJH5WCc-TGYlmCk3izsNRrWyYNo'
|
||||
'g8y77dkxKMjQRGrut-bU6jpu6ctDfOcgTVC-q6sKXURqfNQfP_1ZRCmDApKn3tuTt0vVahZOIJF3rGjvy5bzjyWA'
|
||||
'LBqxnsMxeLEEZnjfzqwG6ElioJjN5tHf4MJFV3I4aCmtSOGsaWfdF8_sYSwZ48Q16UO_-BzE_dE2W2wOQZ2inbut'
|
||||
'IhUU3r-ga_ogGlntLMJ2GD2r2OgCHJ59PHgL-nOnrbfQdZ_sXmcovUnQiAjKuFAx48cQEZ-i74QAbaCgmUFdrDue'
|
||||
'_oWlk76IWXR15mwWLbpBZMR5aG4WREsvroJ17mIabFGi7R2_f7Vpe401W04gqDQjgxhkN0zY4rwBCmWZg6YJNntz'
|
||||
'qgIWoOcc9ZKUEOh-bpyhoWIyFIgtm4YlQQT-nDJwfRt7CCAz9LwTlx8czHn7JoIR4Oj6-fFY06ILYPmn8TOFFCQb'
|
||||
'ZcbdfY-NCE9TU1G60AHddQllarXRSxL4J-MN9tQhk22Ib-cPpGgtSL3L2-12zs0O3SiNUisxdaEEUZpaxauuAu38'
|
||||
'TH1_pelomrwvCdE1ukyyQBKDGJkxf4PxUy298fsg0-RQgALmaKa_ZWl68o5weQyPz-phC9GLjwUCgaKzijoI3fec'
|
||||
'Low1EDYkbCmkGFJhBiaAsl-AexuSKgFwKvTAg-O1R4SYPRShr6laHfzP9A5VC2hrcQ=='
|
||||
),
|
||||
'summary': [],
|
||||
},
|
||||
{
|
||||
'id': 'msg_0b85038a4a37efb6006ae75c0ec8b887d1b503c5d0835acef1',
|
||||
'type': 'message',
|
||||
'status': 'completed',
|
||||
'content': [
|
||||
{
|
||||
'type': 'output_text',
|
||||
'annotations': [],
|
||||
'logprobs': [],
|
||||
'text': (
|
||||
'I’ll read `shop/validation.py` first to check how it handles zero, negative, and non-int'
|
||||
'eger quantities.'
|
||||
),
|
||||
},
|
||||
],
|
||||
'phase': 'commentary',
|
||||
'role': 'assistant',
|
||||
},
|
||||
{
|
||||
'id': 'fc_0b85038a4a37efb6006ae75c0f3af487d1ade6d1cb6b7aab90',
|
||||
'type': 'function_call',
|
||||
'status': 'completed',
|
||||
'arguments': '{"path":"shop/validation.py","start_line":1,"end_line":200}',
|
||||
'call_id': 'call_AJa3dZ4npr0KPN0SEwwl90xc',
|
||||
'name': 'read_file',
|
||||
},
|
||||
],
|
||||
'parallel_tool_calls': True,
|
||||
'presence_penalty': 0.0,
|
||||
'previous_response_id': None,
|
||||
'prompt_cache_key': None,
|
||||
'prompt_cache_retention': '24h',
|
||||
'reasoning': {
|
||||
'context': 'all_turns',
|
||||
'effort': 'high',
|
||||
'mode': 'standard',
|
||||
'summary': None,
|
||||
},
|
||||
'safety_identifier': None,
|
||||
'service_tier': 'default',
|
||||
'store': True,
|
||||
'temperature': 1.0,
|
||||
'text': {
|
||||
'format': {
|
||||
'type': 'text',
|
||||
},
|
||||
'verbosity': 'medium',
|
||||
},
|
||||
'tool_choice': 'auto',
|
||||
'tool_usage': {
|
||||
'image_gen': {
|
||||
'input_tokens': 0,
|
||||
'input_tokens_details': {
|
||||
'image_tokens': 0,
|
||||
'text_tokens': 0,
|
||||
},
|
||||
'output_tokens': 0,
|
||||
'output_tokens_details': {
|
||||
'image_tokens': 0,
|
||||
'text_tokens': 0,
|
||||
},
|
||||
'total_tokens': 0,
|
||||
},
|
||||
'web_search': {
|
||||
'num_requests': 0,
|
||||
},
|
||||
},
|
||||
'tools': [
|
||||
{
|
||||
'type': 'function',
|
||||
'description': 'Search source and test files for a literal term.',
|
||||
'name': 'search_files',
|
||||
'output_schema': None,
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'query': {
|
||||
'type': 'string',
|
||||
},
|
||||
'paths': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'query',
|
||||
'paths',
|
||||
],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
'strict': False,
|
||||
},
|
||||
{
|
||||
'type': 'function',
|
||||
'description': 'Read a range of lines from a project file.',
|
||||
'name': 'read_file',
|
||||
'output_schema': None,
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'path': {
|
||||
'type': 'string',
|
||||
},
|
||||
'start_line': {
|
||||
'type': 'integer',
|
||||
'minimum': 1,
|
||||
},
|
||||
'end_line': {
|
||||
'type': 'integer',
|
||||
'minimum': 1,
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'path',
|
||||
'start_line',
|
||||
'end_line',
|
||||
],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
'strict': False,
|
||||
},
|
||||
],
|
||||
'top_logprobs': 0,
|
||||
'top_p': 0.98,
|
||||
'truncation': 'disabled',
|
||||
'usage': {
|
||||
'input_tokens': 312,
|
||||
'input_tokens_details': {
|
||||
'cache_write_tokens': 0,
|
||||
'cached_tokens': 0,
|
||||
},
|
||||
'output_tokens': 144,
|
||||
'output_tokens_details': {
|
||||
'reasoning_tokens': 79,
|
||||
},
|
||||
'total_tokens': 456,
|
||||
},
|
||||
'user': None,
|
||||
'metadata': {},
|
||||
}
|
||||
|
||||
GPT_6_1_TERRA_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/chat/completions",
|
||||
litellm_request={**CHAT_COMPLETIONS_REQUEST, "model": "openai/gpt-6.1-terra"},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers=GPT_5_6_TERRA_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_request={
|
||||
**OPENAI_RESPONSES_REQUEST_FROM_CHAT_COMPLETIONS,
|
||||
"model": "gpt-6.1-terra",
|
||||
},
|
||||
mock_provider_response=GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE,
|
||||
expected_litellm_response={
|
||||
**GPT_5_6_TERRA_TEST_CASE.expected_litellm_response,
|
||||
"id": "resp_0b85038a4a37efb6006ae75c0cf84c87d1ac49e75264e5701f",
|
||||
"model": "openai/gpt-6.1-terra",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "tool_calls",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "I’ll read `shop/validation.py` first to check how it handles zero, negative, and non-integer quantities.",
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"function": {
|
||||
"arguments": '{"path":"shop/validation.py","start_line":1,"end_line":200}',
|
||||
"name": "read_file",
|
||||
},
|
||||
"id": "call_AJa3dZ4npr0KPN0SEwwl90xc",
|
||||
"type": "function",
|
||||
}
|
||||
],
|
||||
"reasoning_items": [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_0b85038a4a37efb6006ae75c0de1ac87d1a40d2ce0476272d9",
|
||||
"encrypted_content": GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][0][
|
||||
"encrypted_content"
|
||||
],
|
||||
"summary": [],
|
||||
}
|
||||
],
|
||||
"provider_specific_fields": {
|
||||
"phase": "commentary",
|
||||
"openai_responses_replay": ANY,
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 144,
|
||||
"prompt_tokens": 312,
|
||||
"total_tokens": 456,
|
||||
"completion_tokens_details": {"reasoning_tokens": 79},
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 0,
|
||||
"cache_creation_tokens": 0,
|
||||
},
|
||||
},
|
||||
"access_programs": {"cyber": "standard"},
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from integration.translation.chat_completions.bases.openai import (
|
|||
GPT_5_6_LUNA_TEST_CASE,
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_5_6_TERRA_TEST_CASE,
|
||||
GPT_6_1_TERRA_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
GPT_6_ASTRA_TEST_CASE,
|
||||
GPT_6_LUNA_TEST_CASE,
|
||||
|
|
@ -24,6 +25,7 @@ from integration.translation.runner import assert_translation
|
|||
GPT_6_1_SOL_TEST_CASE,
|
||||
GPT_6_ASTRA_TEST_CASE,
|
||||
GPT_5_6_TERRA_TEST_CASE,
|
||||
GPT_6_1_TERRA_TEST_CASE,
|
||||
],
|
||||
ids=lambda case: case.id,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,10 +5,10 @@ Every row sends `CODING_AGENT_REQUEST` to `/v1/chat/completions`, changing only
|
|||
| Tier | Model versions |
|
||||
| --- | --- |
|
||||
| Small | Haiku 4.5 and 5.5, GPT 5 Mini and 5.4 Mini, GPT 5.6 Luna and 6 Luna |
|
||||
| Mid | Opus 5 and 5.5, Sonnet 5.5, Gemini 3.8 Flash and 3.1 Pro Preview, GPT 5.6 Sol, 6 Sol and 6.1 Sol, GPT 5.6 Terra |
|
||||
| Mid | Opus 5 and 5.5, Sonnet 5.5, Gemini 3.8 Flash and 3.1 Pro Preview, GPT 5.6 Sol, 6 Sol and 6.1 Sol, GPT 5.6 and 6.1 Terra |
|
||||
| High | Fable 5 and 5.1, Gemini 4 Argon, GPT 6 Astra |
|
||||
|
||||
The 52 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented
|
||||
The 53 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented
|
||||
|
||||
The request contains system instructions, conversation history, previous assistant tool calls and results, four function definitions, automatic parallel tools, low reasoning effort and an output limit. Each row exercises a tool-call response, raw streaming, SDK stream assembly and a follow-up that replays the returned assistant message and tool-call IDs
|
||||
|
||||
|
|
@ -33,7 +33,7 @@ Streaming cases store `expected_litellm_message` and `expected_litellm_usage`. M
|
|||
|
||||
Usage requires nonnegative integer counts, positive prompt and completion counts, and `total_tokens == prompt_tokens + completion_tokens`. The complete normalized usage must equal the independently expected counters, including cached and reasoning tokens and derived text/cache details. Missing, changed or unexpected counters fail. Only null optional fields, the explicitly listed provider metadata/duplicate counters and computed `cost` are excluded. Thus a wrong prompt count with an adjusted total still fails
|
||||
|
||||
The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 52 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips
|
||||
The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 53 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips
|
||||
|
||||
## Recorded upstreams
|
||||
|
||||
|
|
|
|||
|
|
@ -182,6 +182,11 @@ MID: Final[tuple[AgentCases, ...]] = (
|
|||
openai.GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE,
|
||||
openai.GPT_5_6_TERRA_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
(
|
||||
openai.GPT_6_1_TERRA_AGENT_TEST_CASE,
|
||||
openai.GPT_6_1_TERRA_AGENT_FOLLOWUP_TEST_CASE,
|
||||
openai.GPT_6_1_TERRA_AGENT_STREAMING_TEST_CASE,
|
||||
),
|
||||
(openai.GPT_6_SOL_TEST_CASE, openai.GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE, openai.GPT_6_SOL_AGENT_STREAMING_TEST_CASE),
|
||||
(
|
||||
openai.GPT_6_1_SOL_AGENT_TEST_CASE,
|
||||
|
|
|
|||
|
|
@ -2704,6 +2704,242 @@ GPT_5_6_TERRA_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase(
|
|||
)
|
||||
|
||||
|
||||
def _gpt_6_1_terra_stream_value(value: JsonValue) -> JsonValue:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: {"cyber": "standard"}
|
||||
if key == "access_programs"
|
||||
else 1793547872
|
||||
if key == "created_at"
|
||||
else 1793547874
|
||||
if key == "completed_at" and item is not None
|
||||
else _gpt_6_1_terra_stream_value(item)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_gpt_6_1_terra_stream_value(item) for item in value]
|
||||
if isinstance(value, str):
|
||||
return {
|
||||
"resp_07964387f422fad6006ad04aa9820c87d0b00c0f7e8179e669":
|
||||
"resp_067882b6e08b211f006ae75e608f7c87d1952b1695ca966b80",
|
||||
"fc_07964387f422fad6006ad04aaa99d087d09bf5301d4def0fb9":
|
||||
"fc_067882b6e08b211f006ae75e6284ac87d1854926589ccdeef8",
|
||||
"fc_07964387f422fad6006ad04aaa99e487d0adda87c8afeac85a":
|
||||
"fc_067882b6e08b211f006ae75e62850487d1b7b53aac9602ff43",
|
||||
"call_xDdtGVzn464uEE7X8AuVRiHJ": "call_g1sYS5VaOpk7yFfZWb6C6FhG",
|
||||
"call_yPhpkZWP5B2zJo4SPhCEAxW4": "call_qq6AhKCnaWY7iQsRGkj9tshL",
|
||||
"gpt-5.6-terra": "gpt-6.1-terra",
|
||||
"IFA": "4N0",
|
||||
"RH": "wV",
|
||||
}.get(value, value)
|
||||
return value
|
||||
|
||||
|
||||
GPT_6_1_TERRA_AGENT_TEST_CASE: Final = replace(
|
||||
GPT_5_6_TERRA_AGENT_TEST_CASE,
|
||||
litellm_request={**CODING_AGENT_REQUEST, "model": "openai/gpt-6.1-terra"},
|
||||
expected_provider_request={**OPENAI_RESPONSES_AGENT_REQUEST, "model": "gpt-6.1-terra"},
|
||||
mock_provider_response={
|
||||
**GPT_5_6_TERRA_AGENT_TEST_CASE.mock_provider_response,
|
||||
"id": "resp_04952ef9a70cae0b006ae75e5e7ae487d19a9876588a88de0c",
|
||||
"created_at": 1793547870,
|
||||
"completed_at": 1793547872,
|
||||
"model": "gpt-6.1-terra",
|
||||
"access_programs": {"cyber": "standard"},
|
||||
"output": [
|
||||
{
|
||||
"id": "fc_04952ef9a70cae0b006ae75e5fd56087d19886a8f152af5d41",
|
||||
"type": "function_call",
|
||||
"status": "completed",
|
||||
"arguments": '{"query":"retries_remaining"}',
|
||||
"call_id": "call_EtA7LwkZhItDqLqZtTbMuAMT",
|
||||
"name": "search_code",
|
||||
},
|
||||
{
|
||||
"id": "fc_04952ef9a70cae0b006ae75e5fd57487d1871209cffbada8fb",
|
||||
"type": "function_call",
|
||||
"status": "completed",
|
||||
"arguments": '{"path":"tests/test_retry.py"}',
|
||||
"call_id": "call_lVIvxbSII29GTSNa284H8WhK",
|
||||
"name": "run_tests",
|
||||
},
|
||||
],
|
||||
},
|
||||
expected_litellm_response={
|
||||
**GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response,
|
||||
"id": "resp_04952ef9a70cae0b006ae75e5e7ae487d19a9876588a88de0c",
|
||||
"model": "openai/gpt-6.1-terra",
|
||||
"choices": [
|
||||
{
|
||||
**GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response["choices"][0],
|
||||
"message": {
|
||||
**GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response["choices"][0]["message"],
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"function": {"arguments": '{"query":"retries_remaining"}', "name": "search_code"},
|
||||
"id": "call_EtA7LwkZhItDqLqZtTbMuAMT",
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"function": {"arguments": '{"path":"tests/test_retry.py"}', "name": "run_tests"},
|
||||
"id": "call_lVIvxbSII29GTSNa284H8WhK",
|
||||
"type": "function",
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"access_programs": {"cyber": "standard"},
|
||||
},
|
||||
)
|
||||
|
||||
GPT_6_1_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace(
|
||||
GPT_6_1_TERRA_AGENT_TEST_CASE,
|
||||
scenario="agent_followup",
|
||||
litellm_request=followup_request(
|
||||
GPT_6_1_TERRA_AGENT_TEST_CASE.litellm_request,
|
||||
_without_replay_record(GPT_6_1_TERRA_AGENT_TEST_CASE.expected_litellm_response),
|
||||
),
|
||||
expected_provider_request={
|
||||
**GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_provider_request,
|
||||
"model": "gpt-6.1-terra",
|
||||
"input": [
|
||||
*list_value(OPENAI_RESPONSES_AGENT_REQUEST["input"]),
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_EtA7LwkZhItDqLqZtTbMuAMT",
|
||||
"name": "search_code",
|
||||
"arguments": '{"query":"retries_remaining"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_lVIvxbSII29GTSNa284H8WhK",
|
||||
"name": "run_tests",
|
||||
"arguments": '{"path":"tests/test_retry.py"}',
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_EtA7LwkZhItDqLqZtTbMuAMT",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": '{"matches": [{"path": "src/worker.py", "line": 12, "text": "remaining = '
|
||||
'retries_remaining(3, attempts)"}]}',
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_lVIvxbSII29GTSNa284H8WhK",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": '{"exit_code": 1, "output": "FAILED tests/test_retry.py::test_retries_remaining - '
|
||||
'assert 4 == 2"}',
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "Stop after these diagnostics. Do not call more tools or change files. Summarize the "
|
||||
"bug, the caller found and the failed test in two short sentences.",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
mock_provider_response={
|
||||
**GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.mock_provider_response,
|
||||
"id": "resp_0b23ebd19ad1a426006ae75e8c1de887d1a118f6e1dd5930de",
|
||||
"created_at": 1793547916,
|
||||
"completed_at": 1793547917,
|
||||
"model": "gpt-6.1-terra",
|
||||
"access_programs": {"cyber": "standard"},
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_0b23ebd19ad1a426006ae75e8caa3087d1b36f6b0c2fa883dd",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
"text": "`retries_remaining` adds attempts to the limit instead of subtracting them and clamping the result "
|
||||
"at zero; a caller was found in `src/worker.py`. The test failed because "
|
||||
"`retries_remaining(3, 1)` returned 4 instead of 2.",
|
||||
}
|
||||
],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 619,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 60,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 679,
|
||||
},
|
||||
},
|
||||
expected_litellm_response={
|
||||
**GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response,
|
||||
"id": "resp_0b23ebd19ad1a426006ae75e8c1de887d1a118f6e1dd5930de",
|
||||
"model": "openai/gpt-6.1-terra",
|
||||
"choices": [
|
||||
{
|
||||
**GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response["choices"][0],
|
||||
"message": {
|
||||
**GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response["choices"][0]["message"],
|
||||
"content": "`retries_remaining` adds attempts to the limit instead of subtracting them and clamping "
|
||||
"the result at zero; a caller was found in `src/worker.py`. The test failed because "
|
||||
"`retries_remaining(3, 1)` returned 4 instead of 2.",
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
**GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE.expected_litellm_response["usage"],
|
||||
"completion_tokens": 60,
|
||||
"prompt_tokens": 619,
|
||||
"total_tokens": 679,
|
||||
},
|
||||
"access_programs": {"cyber": "standard"},
|
||||
},
|
||||
)
|
||||
|
||||
GPT_6_1_TERRA_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase(
|
||||
scenario="agent_streaming",
|
||||
litellm_endpoint=GPT_6_1_TERRA_AGENT_TEST_CASE.litellm_endpoint,
|
||||
litellm_request={
|
||||
**GPT_6_1_TERRA_AGENT_TEST_CASE.litellm_request,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
expected_provider_headers=GPT_6_1_TERRA_AGENT_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_request={**OPENAI_RESPONSES_AGENT_REQUEST, "model": "gpt-6.1-terra", "stream": True},
|
||||
mock_provider_frames=tuple(
|
||||
SSEFrame(data=_gpt_6_1_terra_stream_value(frame.data), event=frame.event)
|
||||
for frame in GPT_5_6_TERRA_AGENT_STREAMING_TEST_CASE.mock_provider_frames
|
||||
),
|
||||
expected_litellm_message=coding_agent_stream_message(
|
||||
("call_g1sYS5VaOpk7yFfZWb6C6FhG", "call_qq6AhKCnaWY7iQsRGkj9tshL"),
|
||||
),
|
||||
expected_litellm_usage={
|
||||
"prompt_tokens": 448,
|
||||
"completion_tokens": 53,
|
||||
"total_tokens": 501,
|
||||
"prompt_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0, "cache_creation_tokens": 0},
|
||||
"completion_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
)
|
||||
|
||||
GPT_6_ASTRA_AGENT_TEST_CASE: Final = replace(
|
||||
GPT_6_ASTRA_TEST_CASE,
|
||||
scenario="agent_tools",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from typing import Final
|
||||
from unittest.mock import ANY
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.templates import MESSAGES_BRIDGE_RESPONSE, MESSAGES_REQUEST
|
||||
|
|
@ -100,6 +101,7 @@ GPT_5_4_TEST_CASE: Final = TranslationTestCase(
|
|||
)
|
||||
|
||||
|
||||
|
||||
GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/messages",
|
||||
|
|
@ -784,3 +786,243 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase(
|
|||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE: Final = {
|
||||
'id': 'resp_0a3b11558c0819df006ae75c101de487d1812988a3147f6964',
|
||||
'object': 'response',
|
||||
'created_at': 1793547280,
|
||||
'status': 'completed',
|
||||
'access_programs': {
|
||||
'cyber': 'standard',
|
||||
},
|
||||
'background': False,
|
||||
'billing': {
|
||||
'payer': 'developer',
|
||||
},
|
||||
'completed_at': 1793547282,
|
||||
'error': None,
|
||||
'frequency_penalty': 0.0,
|
||||
'incomplete_details': None,
|
||||
'instructions': (
|
||||
'You help maintain a synthetic Python project. Read the relevant files before suggesting '
|
||||
'a fix. Explain which file you need next, then call a read-only tool. Never change files.'
|
||||
' Use the messages client for this project review. Consider zero, negative and non-intege'
|
||||
'r quantities before choosing the next file to inspect.'
|
||||
),
|
||||
'max_output_tokens': 1024,
|
||||
'max_tool_calls': None,
|
||||
'model': 'gpt-6.1-terra',
|
||||
'moderation': None,
|
||||
'output': [
|
||||
{
|
||||
'id': 'rs_0a3b11558c0819df006ae75c10fde087d192db4aa5ea7cf7be',
|
||||
'type': 'reasoning',
|
||||
'content': [],
|
||||
'encrypted_content': (
|
||||
'gAAAAABq51wSSVCQB1815HYz5pLz1k32-dUjO-X-4b7LQG23OQZx_gqLXLuuL4xBCieTrvYatU55pFIxiQFXgtqG'
|
||||
'6U1VtkToUospW-zTvH4roGicTQdWAovuoGt_w7rIFKNd0DUt8XB82WtVnogSd-ZQi4yJ3fjJLrR8nGTG596tQoGX'
|
||||
'5TIFWeprIbpd99K1-bmu7dupFVg55x8zWfUx2e7wyYRFJ4JZLdIy1_2AwPaf5AjiZn5hnYXdGL3p7X_Bv_nkoRa4'
|
||||
'1EE592bGS0K_GAxxVxH2sAdzQIOQf_G_E4fvnHRegGK61AgZQ9PVn2dXhGCXa-O0j1aYEx5pECMSfHaqExkx5-Nk'
|
||||
'z3l6Y7RmkoUKduquneXcZSRX-COeDTcts0hRFMTnlNQ5obc3tY_IEST-JDpKdsjL-6YEHhtJNrdED_DIj0vVrQgg'
|
||||
'_3g0LfnDGSqtunsHs-KU4He_jYBJwXHNB2xbe5DMVAyNcqm7TOKRDejB_ym6cA_8j-oTXIumSnAao6I-l3WYU2yk'
|
||||
'tprvhby5FPnPhspqCYVu6nn5LxO7wyQOm_bOK4LcCojyp5OGRgCjtZ7AhAytPSryHt_gSMeEmQ11REFYh3PTrPt5'
|
||||
'ZqYYHW-miQ4Kgg80tn2I66EydrVEEaqZp8sQ7XgbFr7CG_6DOVGIGB819hLpA6clAZxNuUc8bVTwvT_o-wOLOePr'
|
||||
'Fa28kMGuucFWgLKW3fxQBh5MVHfMca-T8shc9aK8JLq7H74xfBaNWdXyNpMlNYr_kPC8LHqh2mKXMyG_17yShf-f'
|
||||
'h4MIY7dHrHr15X-aw8tAFBUZAkTTQF9Da4Cn09oX-yGGs1vW75HAPAdaBqVvAu_MAHXn_C3ACA4mKe7N4ihQWcVD'
|
||||
'zw9e0K6tKWwOYybCLenWnlJ64ooPajGtQvdUfYx4SGmcxTPd18q1_djz-J5k7T8gdqwiD5MEdJU3Trsrw7eULiFE'
|
||||
'qJu9oYrgsiVO-lsjIub5MPLCfKb8_L703ls2WBQ64BT-MZw2qgjKdHXk3f9KjRFaWM_HIJgaMrG6x9H2KE5Sqku9'
|
||||
'o1dANEH0ZpxyBb4lIofWnTrrXALN763TCQe0BC0IwmaoJBfAFThzhLbCuQBLsNZj4vi8rC7W6xSBxQ1UYCgSrZPh'
|
||||
'm8JUoGs4RWYgq7-mD3sB2jcRedcoG9RqxpF0h7_DIzHqk3UVnSpNiRnh_wGp4x_-lO2Ms6qp48QpSYOMJmG_KcRI'
|
||||
'6qhcxTg2B3tU4965f2Iz0ghSzRiSctki1CPvWcRBtLv6LqLLoBK0NTgoByR66rUB7s6yYe6coP_OSkR0GFUCqH_X'
|
||||
'w-kcd-VckGJxKUm5tkejTjjrEXq3LiClTn9NUUgmtaWPEmZoHdNElyu1xiIK-0CTCb07zN_EK2_Nls5SRsWZBRFt'
|
||||
'3YCGcw9vK2jSyx28cRwdBmb1fWXAQ3FYAl1uISH5kdcajQvWv1rhHPMW828OYZTZxG1whGwNPxOTWwj2sHuxvMwG'
|
||||
'MmOnhdTnIX-5qoNafrWraIv8jBC4Y4Z_hlCaB89Ba4Y_-B5-0ClNLGOx6q9fzj3wY4H3RAymTbi9pqqUpuX7wWBG'
|
||||
'uL9KggI6a5A09tmTIpucIsHY7dH6c12Xrls7vkrxlQh_7iix67VUj9UmspFvnCSVmX0W3IdPszNZpVHNpm_KOD-v'
|
||||
'Y5fSNhkTk_hTjjlSMhSe6Ki7BarnMzPo8j7EHVZcgmoAOMoNDHvIEPjbCSXphGr9qYfQ4ajffaloJLh-51e0C8mJ'
|
||||
'1daJ2xs7ZT7uJI0syeeKVRnqxDYlvoAEKl1KrUrX9SKceoVANOla-DMWs3TSsUL6-ccPqzJFDdPn6aVoIxS4IMGa'
|
||||
'nn5oWXVRxLUS01d19mVPn_XoNXrLwi2GCW0r8GfowHyeEgW535xwNt5uEGmmmV17baclC6x1GUo3xPkKpnPaVgf7'
|
||||
'jIjBP7P8-FOcHoIFB9MfUL_Ot2UdK26tc7vL7CgUuch08WPZOklx8Vsv7N0OIzWiczrA09il52Z_QyyTB5VmfiYS'
|
||||
'X6cLsnFCTh7fTSDs4p80vEfBWqKCW3M0isAWLTEaQ7kwpmHulUI_fvxJg2Bn-lV48aKaLeZT0FAS_WRP-cGBIb79'
|
||||
's8TlmnRcyBJhR0LbMEyMHqo4Y7hwJAyTZkBtejuCyyE3b-2QKq2w-GtcgD5MI9b6EOt7ET6EbFFS'
|
||||
),
|
||||
'summary': [],
|
||||
},
|
||||
{
|
||||
'id': 'msg_0a3b11558c0819df006ae75c11a3c087d18ee211f7533e565f',
|
||||
'type': 'message',
|
||||
'status': 'completed',
|
||||
'content': [
|
||||
{
|
||||
'type': 'output_text',
|
||||
'annotations': [],
|
||||
'logprobs': [],
|
||||
'text': (
|
||||
'I’ll read `shop/validation.py` first, checking how it handles zero, negative, and non-in'
|
||||
'teger quantities.'
|
||||
),
|
||||
},
|
||||
],
|
||||
'phase': 'commentary',
|
||||
'role': 'assistant',
|
||||
},
|
||||
{
|
||||
'id': 'fc_0a3b11558c0819df006ae75c12142887d18b71a86e0c02ae8c',
|
||||
'type': 'function_call',
|
||||
'status': 'completed',
|
||||
'arguments': '{"path":"shop/validation.py","start_line":1,"end_line":150}',
|
||||
'call_id': 'call_Fnk0hANehtgpqZaZieVoZUYK',
|
||||
'name': 'read_file',
|
||||
},
|
||||
],
|
||||
'parallel_tool_calls': True,
|
||||
'presence_penalty': 0.0,
|
||||
'previous_response_id': None,
|
||||
'prompt_cache_key': None,
|
||||
'prompt_cache_retention': '24h',
|
||||
'reasoning': {
|
||||
'context': 'all_turns',
|
||||
'effort': 'high',
|
||||
'mode': 'standard',
|
||||
'summary': None,
|
||||
},
|
||||
'safety_identifier': None,
|
||||
'service_tier': 'default',
|
||||
'store': True,
|
||||
'temperature': 1.0,
|
||||
'text': {
|
||||
'format': {
|
||||
'type': 'text',
|
||||
},
|
||||
'verbosity': 'medium',
|
||||
},
|
||||
'tool_choice': 'auto',
|
||||
'tool_usage': {
|
||||
'image_gen': {
|
||||
'input_tokens': 0,
|
||||
'input_tokens_details': {
|
||||
'image_tokens': 0,
|
||||
'text_tokens': 0,
|
||||
},
|
||||
'output_tokens': 0,
|
||||
'output_tokens_details': {
|
||||
'image_tokens': 0,
|
||||
'text_tokens': 0,
|
||||
},
|
||||
'total_tokens': 0,
|
||||
},
|
||||
'web_search': {
|
||||
'num_requests': 0,
|
||||
},
|
||||
},
|
||||
'tools': [
|
||||
{
|
||||
'type': 'function',
|
||||
'description': 'Search source and test files for a literal term.',
|
||||
'name': 'search_files',
|
||||
'output_schema': None,
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'query': {
|
||||
'type': 'string',
|
||||
},
|
||||
'paths': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'query',
|
||||
'paths',
|
||||
],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
'strict': False,
|
||||
},
|
||||
{
|
||||
'type': 'function',
|
||||
'description': 'Read a range of lines from a project file.',
|
||||
'name': 'read_file',
|
||||
'output_schema': None,
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'path': {
|
||||
'type': 'string',
|
||||
},
|
||||
'start_line': {
|
||||
'type': 'integer',
|
||||
'minimum': 1,
|
||||
},
|
||||
'end_line': {
|
||||
'type': 'integer',
|
||||
'minimum': 1,
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'path',
|
||||
'start_line',
|
||||
'end_line',
|
||||
],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
'strict': False,
|
||||
},
|
||||
],
|
||||
'top_logprobs': 0,
|
||||
'top_p': 0.98,
|
||||
'truncation': 'disabled',
|
||||
'usage': {
|
||||
'input_tokens': 309,
|
||||
'input_tokens_details': {
|
||||
'cache_write_tokens': 0,
|
||||
'cached_tokens': 0,
|
||||
},
|
||||
'output_tokens': 121,
|
||||
'output_tokens_details': {
|
||||
'reasoning_tokens': 56,
|
||||
},
|
||||
'total_tokens': 430,
|
||||
},
|
||||
'user': None,
|
||||
'metadata': {},
|
||||
}
|
||||
|
||||
GPT_6_1_TERRA_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/messages",
|
||||
litellm_request={**MESSAGES_REQUEST, "model": "openai/gpt-6.1-terra"},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers=GPT_5_6_TERRA_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_request={
|
||||
**OPENAI_RESPONSES_REQUEST_FROM_MESSAGES,
|
||||
"model": "gpt-6.1-terra",
|
||||
},
|
||||
mock_provider_response=GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE,
|
||||
expected_litellm_response={
|
||||
**GPT_5_6_TERRA_TEST_CASE.expected_litellm_response,
|
||||
"id": ANY,
|
||||
"content": [
|
||||
{
|
||||
"type": "redacted_thinking",
|
||||
"data": "litellm_encrypted_reasoning:"
|
||||
+ GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][0]["encrypted_content"],
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I’ll read `shop/validation.py` first, checking how it handles zero, negative, and non-integer quantities.",
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "call_Fnk0hANehtgpqZaZieVoZUYK",
|
||||
"name": "read_file",
|
||||
"input": {"path": "shop/validation.py", "start_line": 1, "end_line": 150},
|
||||
},
|
||||
],
|
||||
"usage": {"input_tokens": 309, "output_tokens": 121},
|
||||
"model": "openai/gpt-6.1-terra",
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from integration.translation.messages.bases.openai import (
|
|||
GPT_5_6_LUNA_TEST_CASE,
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_5_6_TERRA_TEST_CASE,
|
||||
GPT_6_1_TERRA_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
GPT_6_ASTRA_TEST_CASE,
|
||||
GPT_6_LUNA_TEST_CASE,
|
||||
|
|
@ -24,6 +25,7 @@ from integration.translation.runner import assert_translation
|
|||
GPT_6_1_SOL_TEST_CASE,
|
||||
GPT_6_ASTRA_TEST_CASE,
|
||||
GPT_5_6_TERRA_TEST_CASE,
|
||||
GPT_6_1_TERRA_TEST_CASE,
|
||||
],
|
||||
ids=lambda case: case.id,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,78 @@ SIGNATURE_1: Final = (
|
|||
"1FW7qRmOMOKJKhbheeMpsTs7XvdvsiDQqgM4PAJt4cwgGAE="
|
||||
)
|
||||
|
||||
CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_1: Final = (
|
||||
"CAQS2gkKEAgTGAI4AUIIdGhpbmtpbmcSDCKQrdbAsNSlkOf3zBoMeduhYaXR7h2XbdAKIjAtYK1x82HeQuy3OMDRyk8Ru1NL6Bo2p4jMGpeqJSUAqMBl"
|
||||
"hxlFnMpq2arnT1bWs3kq9wjg19gHCDS58DA+DUr3YdYx6gy1aX/Qn1XeFVB1L/kSAHlK0Wuxzn41ZM3D3xdGPeg9El2AIQryUlRs/POxJdqRzXes2H5U"
|
||||
"fUU1f/azCoeDJv9RRk234jIMkgBCWiLstzExNwxL+kvTlY2mrQXQ+U9mhXRJC90AilZlFIY8fuRhL8HMFA6d0rqMisr3wLd9M7ACwVP3cOZXx1Wa9Sfb"
|
||||
"2yiutDHkG4fI61xE8cDTYvOqtyY6ZQ9juiV6/rOPDSP+85sLVVmH+wJUJXIEfYBRCmytchPqR/Y/Q1EdMnmiLYfssjWsj8NkfsEBjOHdlIYDTzvplkEK"
|
||||
"y2SHSBwQeHcahHr1Y1napFobIRjVuQnPOMGvhNLUJZwN4zJLAQXzxO5hK2KrFOg66nX3A/+77CGVUr+CpC+/Wbkyc7LQNfI+xzsdRjC7epieVPbgVFDS"
|
||||
"Rt0a7nppVA7JbYzZh04UdCoD7z2dDvX3lhE3WDdyICUatVgsRaU0w1Qmiu9fOwrfU6I29PaXDGw1i4pth6uL8GAS/ZuUC6xFhggEoXs497MgrD3N0uLQ"
|
||||
"69x+IGmuRzIPbcgWjC3IlNMI2se/FpfWXJAsScnCVWRqfBL8crjsmdbuwy4L/usMNT87H3vngSbpd9dZuKpKBW18Y+OPLQiF2im/Pv+zZpA1RJkOvXPr"
|
||||
"ru8eiHu9z2XKVIJgdxKkRjLgi4/JVnN54q3bdkSz7KARIiRc4EzKsyU3/PzJJN1Fev10hxbU4V+Pkv6hLcjzIL9pMGAYjZXrjpxyw+PTVt7jK2k1C1SB"
|
||||
"BXpHdmntJJ30E94IpSISyNIW9Twi3KPYcYhj/rd9WA5UwvhDtbvril6ET10JlqzV36l8RxWUPhlXegJgtEMdnjNplhfDeGsMwg8e3OL1GNRUQoqQNd1Q"
|
||||
"jP3bpHx0yaxaRlkGwgtIGXj2bxuwLwn5otMEMGzQEnnQVNHdjAIpuHunLH9k01S86ir6s5cP05+U1Y1V1U4dZiFVnr4vSwWLIZLpRg2k/O4/inqzIjOY"
|
||||
"LD9xE+d+Mm55vKnbYtIv+DM1GHpKdvbzSfa1czjV6YT1PKotUZpg4ul+uyQqZYhpq9dGYUttevBz5tPIs+XUL6nx3Kj8WnPtkaAD+MuH0+yuALNdwzGb"
|
||||
"4sxxAu4sYsbxHCzpriX+JLjgUq2bO2Y7ve1nHVsh1B1bzyatgv9hnAQyO7uLMP0IICF3F43zJtgol3/gng5bf1T6EeFHDyZAUxGAqbWGjPwh5awEfdLe"
|
||||
"SYc2AtK84ieDLU0LfhvuGRdmAcVXrGmF1D/mhHg95Hs8fKt5QN7O1/gUSsC0TvXGrpbYeE5OH/O2aFFzimURUyUp7W8f6UFNNy4ZHIjdUqomnUL8XH+o"
|
||||
"Rf69GBqbv8BsfPQuq9D6nAvFk+suLYhLvWdFRy/sMA/gXh2mq21t2bFXQXOjgq/dQ7GUJx0vU86alotjdPovkksZAqzHi1s5h3WrQ1h3KxF4pfVO309h"
|
||||
"lQiFvg3fNPaDARqFFci+7ToS0kQXk4r95A6XzGMYAQ=="
|
||||
)
|
||||
|
||||
CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_2: Final = (
|
||||
"CAQS1gcKEQgTGAI4AUIJbmFycmF0aW9uEgxdyD7sRZxxSBB6HUEaDB0dBrarQ/maHvvYRiIwNo20xJqSuL5tpy0fFBXRggkfQd3MO+ehcdkzkAP1bNQF"
|
||||
"I7P1/rfhUbkzeiZI7NDDKvIG0xpIrPlf76ufYrZ2d9+04GJUImemCjrKGxjniCVaBpqVqJp0K7qWu24HZqZjX/pcKUi//W0t73h1BFmkUEy/8XSvsuUs"
|
||||
"u7lJqmMXACCXhkXZhg/EnswdEzKhYwVDay6cGqfGUUo+NBTmhMjYjy69h7x1+RPuEWTNNMhusS2K9U1hFwZjqEdjo/4CsWZzZC6/9WC213voHd6ewjly"
|
||||
"QQLjEKJgoQ4Mnlyep7NVe3b4Hzu0pChITa/PUyhEGEoQhIZoMx3/hYAf3HwqveaPBQRmrjf8m3bvcbh7R0hTJHS3dpUrz5RE6d8zSy6EGWTDTYqb6JpQ"
|
||||
"DdL0+5UxQMZEc6JTcpcBpfxJvS2KFyxyZwOkt02OdjEoTkPxaQTK9fnjVSq9xp5qLm/RC03XnBdKBG5Elk/Dxj4UqPphkKN+UQCyYKivCpkU/N0/MVm1"
|
||||
"/Rp0x5OQuZ/xVSbdyuF8ijlmK/FZuZ8P3sR0v5gvfPGvDoijdWaljc/+ZiJ7CK9LtgoSk16IYrCbKtbL75S7yGTHnWgnHSUzuXorpkf5xGuGKucloLTf"
|
||||
"DPaBsxw7mWUEYcoUE1LxUqip9j2p6Zf6BkOHX5TPgXB14F6194/vwIfoNV7QK/uXFT2kZ/6m4YDfTzmvDCtUP9PcfbyjOIPHXukL/KzbaunT91spQf28"
|
||||
"7CfAbZp0Tl0ATcVut63UB/3e26zxhZ45ZOLyjahllcLY43FWmPLM+tTUPtNmKQ99+JVBS8A4rqKZwnovgFJG4wmjbeBClY691hNoNVsIfPQs0RM0zgIh"
|
||||
"F4rutOqnPIBbcS/5eFGucSB3sAplbqDTQtHVIagaTFAgLtEBPxGqb9jze+Wwwt2RQ9xXE36bEEsgL+jgrd7CfiTs78RCV/G6tKzyfO5TJjQoX/8XUIag"
|
||||
"C5zdRpNg7cTRcDIZPkTl0Jii/Pgrhd3xkBG5heg9YOnsm1xuHkmTjGGGx1sDMeBfLiRPNyYTDHVAqejbqB2dyGLvfsx3DZgSLnv4IBXcesQ0am673hqL"
|
||||
"WYYMT7md2Z9JKoYGnTXqkbmXDkuf6H+RasWhGYkNJtF+XBxUwHVvbMjQaO2tJpQhv8msR3yORQn2Lnt5OF2ws69Irqt5tdkQCyCGn+O6xxOftE3gmEMZ"
|
||||
"0Udpgod2UvJj1Vz2udQhXAkmNRmPgLmp6jRWU7bmGAE="
|
||||
)
|
||||
|
||||
CLAUDE_SONNET_5_5_THINKING_BUDGET_CONTENT: Final = [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "",
|
||||
"signature": CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_1,
|
||||
},
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "",
|
||||
"signature": CLAUDE_SONNET_5_5_THINKING_BUDGET_SIGNATURE_2,
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01CndWUrVgzs4gR9gg1R6w44",
|
||||
"name": "read_file",
|
||||
"input": {"path": "shop/validation.py", "start_line": 1, "end_line": 60},
|
||||
"caller": {"type": "direct"},
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_014RmKHkVks5GvQY6qZoXboC",
|
||||
"name": "read_file",
|
||||
"input": {"path": "tests/test_validation.py", "start_line": 1, "end_line": 40},
|
||||
"caller": {"type": "direct"},
|
||||
},
|
||||
]
|
||||
|
||||
CLAUDE_SONNET_5_5_THINKING_BUDGET_USAGE: Final = {
|
||||
"input_tokens": 4,
|
||||
"cache_creation_input_tokens": 873,
|
||||
"cache_read_input_tokens": 0,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 873, "ephemeral_1h_input_tokens": 0},
|
||||
"output_tokens": 449,
|
||||
"output_tokens_details": {"thinking_tokens": 253},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
}
|
||||
|
||||
CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace(
|
||||
CLAUDE_SONNET_4_6_TEST_CASE,
|
||||
scenario="thinking_budget",
|
||||
|
|
@ -74,9 +146,44 @@ CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE: Final = replace(
|
|||
},
|
||||
)
|
||||
|
||||
CLAUDE_SONNET_5_5_THINKING_BUDGET_TEST_CASE: Final = replace(
|
||||
CLAUDE_SONNET_5_5_TEST_CASE,
|
||||
scenario="thinking_budget",
|
||||
litellm_request={
|
||||
**CLAUDE_SONNET_5_5_TEST_CASE.litellm_request,
|
||||
"max_tokens": 2048,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 1024},
|
||||
},
|
||||
expected_provider_request={
|
||||
**CLAUDE_SONNET_5_5_TEST_CASE.expected_provider_request,
|
||||
"max_tokens": 2048,
|
||||
"thinking": {"type": "adaptive"},
|
||||
},
|
||||
mock_provider_response={
|
||||
**CLAUDE_SONNET_5_5_TEST_CASE.mock_provider_response,
|
||||
"id": "msg_011CgcUJqytdPvGPzVpJNKEQ",
|
||||
"content": CLAUDE_SONNET_5_5_THINKING_BUDGET_CONTENT,
|
||||
"usage": CLAUDE_SONNET_5_5_THINKING_BUDGET_USAGE,
|
||||
},
|
||||
expected_litellm_response={
|
||||
**CLAUDE_SONNET_5_5_TEST_CASE.expected_litellm_response,
|
||||
"id": "msg_011CgcUJqytdPvGPzVpJNKEQ",
|
||||
"content": CLAUDE_SONNET_5_5_THINKING_BUDGET_CONTENT,
|
||||
"usage": CLAUDE_SONNET_5_5_THINKING_BUDGET_USAGE,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", [CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE], ids=(CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE.id,)
|
||||
"case",
|
||||
[
|
||||
CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE,
|
||||
CLAUDE_SONNET_5_5_THINKING_BUDGET_TEST_CASE,
|
||||
],
|
||||
ids=(
|
||||
CLAUDE_SONNET_4_6_THINKING_BUDGET_TEST_CASE.id,
|
||||
CLAUDE_SONNET_5_5_THINKING_BUDGET_TEST_CASE.id,
|
||||
),
|
||||
)
|
||||
def test_messages_reasoning_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
|
|||
|
|
@ -161,6 +161,7 @@ GPT_5_4_TEST_CASE: Final = TranslationTestCase(
|
|||
)
|
||||
|
||||
|
||||
|
||||
GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/responses",
|
||||
|
|
@ -1158,3 +1159,256 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase(
|
|||
"top_logprobs": 0,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE: Final = {
|
||||
'id': 'resp_0489dbceb3070b9a006ae75c12ef5487d1a7cb64a92d8774d1',
|
||||
'object': 'response',
|
||||
'created_at': 1793547282,
|
||||
'status': 'completed',
|
||||
'access_programs': {
|
||||
'cyber': 'standard',
|
||||
},
|
||||
'background': False,
|
||||
'billing': {
|
||||
'payer': 'developer',
|
||||
},
|
||||
'completed_at': 1793547285,
|
||||
'error': None,
|
||||
'frequency_penalty': 0.0,
|
||||
'incomplete_details': None,
|
||||
'instructions': (
|
||||
'You help maintain a synthetic Python project. Read the relevant files before suggesting '
|
||||
'a fix. Explain which file you need next, then call a read-only tool. Never change files.'
|
||||
' Use the responses client for this project review. Consider zero, negative and non-integ'
|
||||
'er quantities before choosing the next file to inspect.'
|
||||
),
|
||||
'max_output_tokens': 1024,
|
||||
'max_tool_calls': None,
|
||||
'model': 'gpt-6.1-terra',
|
||||
'moderation': None,
|
||||
'output': [
|
||||
{
|
||||
'id': 'rs_0489dbceb3070b9a006ae75c13f80487d1ab5a5153f0461603',
|
||||
'type': 'reasoning',
|
||||
'content': [],
|
||||
'encrypted_content': (
|
||||
'gAAAAABq51wVlJWN-LTJqqhWvFyWjY49-yFum5T56W8eITsd-qT0V290Gw80RoL9VVX4jvv6cfpCOb73lk6WHbMW'
|
||||
'q0tjO3yeVbczh_0YDEhG4-CGvQaElquYG1AK2fABB3X_J3xPglgIB9OnPuQiLoYBQHqYy3U-0TinuAr0lFCcWFex'
|
||||
'r1JAcM5JExGND9kJUCXx6Zfpt1_e4M467kCKmYW_pZ3Y0Ndf5uFLooX18Xf9-GBGiB6TtaS_atSGAVaeBYFYaNc4'
|
||||
'WIaTG-675Lnd6dHGQIrV4jBW7Xf4QKlQU1hFZCkUg0JW54skR8c7iMdA8qvdo2ieZ6JgnxU0gujk3OycBhXM1SaL'
|
||||
'P-MenI0QsOFcWASEozr-U53h-b4Gek9TBwY1PGMA8hITFw7tEwpt-s9LvZuDr0kcknR5d8RYyf1owYV-O668ykR5'
|
||||
'VUzheo2NOVUzQIe9xQorENDnxrdya8gO54HxDV9ShzbZ9e9ymv9iRH6jCjSaTOXJdCPjfxd5aYoPTnfibh0RYxM2'
|
||||
'UlcMZhpv5aXfu7c7hUbXFKx1eky48LAE0s1DjcVGa0YCqFNWAJEYJR7ZHpcaN8dzAc4HwNZZo2Oh15AfybRkm8Zh'
|
||||
'Jb5EalmOETrLK0_jxlH29Ln4xqpwlBHWeq6l0z_Qt3CqihV4dV5gsRGaJky6NoSn7UjpWaCzAkFjf1R4nIvqsJDM'
|
||||
'CmFAlS32ne_nL9spLd3uUAXzMCLwnfCs30xp8XxDwFwYq7LweRjdBkEBeOibTsaACR_b4Cp5GwICWsqrzXRpSh7K'
|
||||
'w73iewX7dq6gOiXZlf3bFg6A35h2EIVUPwBnEViN-mwgebe1pLH3vFUHocR3K70PIs7qLlWE5-ZsX_qVd_E1EmbR'
|
||||
'zom8leGZ798zodGpJyy-aTI49RgsPwk4gWugaoB8MKI3ykkmAeLbF3pDRvzTg3ZxF49fFS-0ALrQhB2177MZWymu'
|
||||
'ZHh_SQFi12KCMhzXxO8GkzeMDFV0OKezVkeveafa1eRfWReJJqaobBWd705vLaqidOTIYErHGVhEDKL9judPl0nl'
|
||||
'UhPXh4lDw7LN1qVXUiTFateiic7m-y-ibeTJj500tooNOJG4BSlmHYgZF1hA1FUEtQlDWXFbTTcz8yTXLB0ucbXb'
|
||||
'GudNKyI6u2oTPn0kaZvHuw5tra6_Lkp2G1DJ9ZL-tMLRLH8Fca4RzVTJnoqHZ2S-rl7ukKcYKQaar5ZE4I2zL_F7'
|
||||
'n_4Zr6lIlKOapXkcvso5UpThSy6OnMcUVh9DqWBKcCcQRA3TSshiQHNuY8xQIoNdm7YL4pRzufuvykbINnfUS0UD'
|
||||
'RUOSgNGImQHwNggEJysPx9ctqRXW0hpvcF5wrZbAtGLa6Eun-mBQ_z6-b7vT_uIJmzD9PWoaMfcHEb5fa4nXVFsx'
|
||||
'xBIvlMZ8cM7Yims2aG9PT1V7UrsUyVPP_K8fHJdheOM4oy4nC77fYrEjSUJeh-uCJ6wbXE0PuNBQWivWcwE9VV7U'
|
||||
'jXiIsCEblEVGHlhKL2EEajfB0tEYDOTVV4B2agwwLU5S6JX91UCpcH1dvFTYS_roXMDf3CWmo0JseRW2oZzxcCoP'
|
||||
'NZVvGEMhTCsitwVUEaRBPfNtLiplhbOrUx80IY38kRld9OQb6KxOvyuxqG4LT4_afTQImSGEZJQSW8qo9GzKN_sJ'
|
||||
'9iQvGw6tgqrwqmmLKXjFDp3I5FB29xZAOoxhnapGIKL6KUuehubY_kM5ok21PvPuvfsY-gaFzFQB5dJsaBNZoDaJ'
|
||||
'4cB1pw-tXmP4gUECk0PkoMQEekSzBJLfwk7jbIMS_YSzpE1RcqFenprMVkq-HBfAaaKIQxRi2qB3WM2VaoYGDDFG'
|
||||
'Xi3E3lc6x0D64-vXtxo4EPK-eXPC-ThPShl6POgKuoQZjDvo918hJDn3PCBq7jVXeEAgpOvLkSWQnen_khH5y4Un'
|
||||
'tF8L1auZfD2YugvETRdQsQKuX5ZCJ0_gT9L2Olc3QXbZoE4wzUBiyBdLlysDZanR4e3bgff3ZcsH02Et9iO2ljbO'
|
||||
'8ELqlEzfZWV6lU_MT8GtHJfeIzDPp4UDraHdwyQYtkbJ9LNaub0JR4sushyYO6fLZLB45GqP8TJtCVlorF6okoy4'
|
||||
'AAqBvLb9vu2PAD8a9pjfZ6WNypYJelZ9eEsBTQ_sZGWnBFmvNhjZbFoXHkNAgp9HL0msO7ydRPnL_BPytOMXHiqy'
|
||||
'5VcVaLnRCno3wm-QftrJyrp8AQKnjZL8DEXAYUUrAGofQ5wFA7HhjKGPTTLxd-cahYh9Lkkfwp3fUj3YgPTYDM35'
|
||||
'I4dpgUG9UCutIYwtmgdM-uKiot2w'
|
||||
),
|
||||
'summary': [],
|
||||
},
|
||||
{
|
||||
'id': 'msg_0489dbceb3070b9a006ae75c14b9d487d183c51260c69dd557',
|
||||
'type': 'message',
|
||||
'status': 'completed',
|
||||
'content': [
|
||||
{
|
||||
'type': 'output_text',
|
||||
'annotations': [],
|
||||
'logprobs': [],
|
||||
'text': (
|
||||
'I’ll read `shop/validation.py` first to check how it handles zero, negative, and non-int'
|
||||
'eger quantities.'
|
||||
),
|
||||
},
|
||||
],
|
||||
'phase': 'commentary',
|
||||
'role': 'assistant',
|
||||
},
|
||||
{
|
||||
'id': 'fc_0489dbceb3070b9a006ae75c152c9887d19f48a7ea1647f1e4',
|
||||
'type': 'function_call',
|
||||
'status': 'completed',
|
||||
'arguments': '{"path":"shop/validation.py","start_line":1,"end_line":200}',
|
||||
'call_id': 'call_J39X3VwRp2iE5zJYYELJRTOn',
|
||||
'name': 'read_file',
|
||||
},
|
||||
],
|
||||
'parallel_tool_calls': True,
|
||||
'presence_penalty': 0.0,
|
||||
'previous_response_id': None,
|
||||
'prompt_cache_key': None,
|
||||
'prompt_cache_retention': '24h',
|
||||
'reasoning': {
|
||||
'context': 'all_turns',
|
||||
'effort': 'high',
|
||||
'mode': 'standard',
|
||||
'summary': None,
|
||||
},
|
||||
'safety_identifier': None,
|
||||
'service_tier': 'default',
|
||||
'store': True,
|
||||
'temperature': 1.0,
|
||||
'text': {
|
||||
'format': {
|
||||
'type': 'text',
|
||||
},
|
||||
'verbosity': 'medium',
|
||||
},
|
||||
'tool_choice': 'auto',
|
||||
'tool_usage': {
|
||||
'image_gen': {
|
||||
'input_tokens': 0,
|
||||
'input_tokens_details': {
|
||||
'image_tokens': 0,
|
||||
'text_tokens': 0,
|
||||
},
|
||||
'output_tokens': 0,
|
||||
'output_tokens_details': {
|
||||
'image_tokens': 0,
|
||||
'text_tokens': 0,
|
||||
},
|
||||
'total_tokens': 0,
|
||||
},
|
||||
'web_search': {
|
||||
'num_requests': 0,
|
||||
},
|
||||
},
|
||||
'tools': [
|
||||
{
|
||||
'type': 'function',
|
||||
'description': 'Search source and test files for a literal term.',
|
||||
'name': 'search_files',
|
||||
'output_schema': None,
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'query': {
|
||||
'type': 'string',
|
||||
},
|
||||
'paths': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'query',
|
||||
'paths',
|
||||
],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
'strict': True,
|
||||
},
|
||||
{
|
||||
'type': 'function',
|
||||
'description': 'Read a range of lines from a project file.',
|
||||
'name': 'read_file',
|
||||
'output_schema': None,
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'path': {
|
||||
'type': 'string',
|
||||
},
|
||||
'start_line': {
|
||||
'type': 'integer',
|
||||
'minimum': 1,
|
||||
},
|
||||
'end_line': {
|
||||
'type': 'integer',
|
||||
'minimum': 1,
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'path',
|
||||
'start_line',
|
||||
'end_line',
|
||||
],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
'strict': True,
|
||||
},
|
||||
],
|
||||
'top_logprobs': 0,
|
||||
'top_p': 0.98,
|
||||
'truncation': 'disabled',
|
||||
'usage': {
|
||||
'input_tokens': 309,
|
||||
'input_tokens_details': {
|
||||
'cache_write_tokens': 0,
|
||||
'cached_tokens': 0,
|
||||
},
|
||||
'output_tokens': 131,
|
||||
'output_tokens_details': {
|
||||
'reasoning_tokens': 66,
|
||||
},
|
||||
'total_tokens': 440,
|
||||
},
|
||||
'user': None,
|
||||
'metadata': {},
|
||||
}
|
||||
|
||||
GPT_6_1_TERRA_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/responses",
|
||||
litellm_request={**RESPONSES_REQUEST, "model": "openai/gpt-6.1-terra"},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers=GPT_5_6_TERRA_TEST_CASE.expected_provider_headers,
|
||||
expected_provider_request={
|
||||
**OPENAI_RESPONSES_REQUEST,
|
||||
"model": "gpt-6.1-terra",
|
||||
},
|
||||
mock_provider_response=GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE,
|
||||
expected_litellm_response={
|
||||
**GPT_5_6_TERRA_TEST_CASE.expected_litellm_response,
|
||||
"created_at": 1793547282,
|
||||
"completed_at": 1793547285,
|
||||
"model": "openai/gpt-6.1-terra",
|
||||
"access_programs": {"cyber": "standard"},
|
||||
"output": [
|
||||
*GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][:2],
|
||||
{
|
||||
**GPT_6_1_TERRA_MOCK_PROVIDER_RESPONSE["output"][2],
|
||||
"namespace": None,
|
||||
},
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 309,
|
||||
"input_tokens_details": {
|
||||
"audio_tokens": None,
|
||||
"cached_tokens": 0,
|
||||
"cached_tokens_details": None,
|
||||
"image_tokens": None,
|
||||
"text_tokens": None,
|
||||
"video_tokens": None,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"output_tokens": 131,
|
||||
"output_tokens_details": {
|
||||
"audio_tokens": None,
|
||||
"reasoning_tokens": 66,
|
||||
"text_tokens": None,
|
||||
},
|
||||
"total_tokens": 440,
|
||||
"cost": None,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from integration.translation.responses.bases.openai import (
|
|||
GPT_5_6_LUNA_TEST_CASE,
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_5_6_TERRA_TEST_CASE,
|
||||
GPT_6_1_TERRA_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
GPT_6_ASTRA_TEST_CASE,
|
||||
GPT_6_LUNA_TEST_CASE,
|
||||
|
|
@ -24,6 +25,7 @@ from integration.translation.runner import assert_translation
|
|||
GPT_6_1_SOL_TEST_CASE,
|
||||
GPT_6_ASTRA_TEST_CASE,
|
||||
GPT_5_6_TERRA_TEST_CASE,
|
||||
GPT_6_1_TERRA_TEST_CASE,
|
||||
],
|
||||
ids=lambda case: case.id,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue