diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 85ada90ec45..4dd11f416d7 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -110,6 +110,31 @@ model_list: api_base: http://127.0.0.1:8191 api_key: synthetic-bedrock-key aws_region_name: us-east-1 + - model_name: openai/gpt-5.4 + litellm_params: + model: openai/gpt-5.4 + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key + - model_name: openai/gpt-5.6-sol + litellm_params: + model: openai/gpt-5.6-sol + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key + - model_name: openai/gpt-5.6-luna + litellm_params: + model: openai/gpt-5.6-luna + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key + - model_name: openai/gpt-6-luna + litellm_params: + model: openai/gpt-6-luna + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key + - model_name: openai/gpt-6.1-sol + litellm_params: + model: openai/gpt-6.1-sol + api_base: http://127.0.0.1:8191 + api_key: synthetic-openai-key - model_name: openai/responses/gpt-5.6-sol litellm_params: model: openai/responses/gpt-5.6-sol diff --git a/tests/integration/translation/chat_completions/bases/openai.py b/tests/integration/translation/chat_completions/bases/openai.py new file mode 100644 index 00000000000..8b712352c29 --- /dev/null +++ b/tests/integration/translation/chat_completions/bases/openai.py @@ -0,0 +1,463 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + + +GPT_5_4_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "openai/gpt-5.4", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/chat/completions", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.4", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "id": "chatcmpl-EVhxnsh4umJNE0UBHCO0G3rCQjLO9", + "object": "chat.completion", + "created": 1791225871, + "model": "gpt-5.4-2026-03-05", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello.", "refusal": None, "annotations": []}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 19, + "completion_tokens": 5, + "total_tokens": 24, + "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0}, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, + expected_litellm_response={ + "id": "chatcmpl-EVhxnsh4umJNE0UBHCO0G3rCQjLO9", + "created": 1791225871, + "model": "openai/gpt-5.4", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello.", + "role": "assistant", + "refusal": None, + "annotations": [], + }, + "provider_specific_fields": {}, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, + }, + "service_tier": "default", + "system_fingerprint": None, + }, +) + + +GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "openai/gpt-5.6-sol", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/chat/completions", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.6-sol", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "id": "chatcmpl-EVhxr75iujjmwE0F2bgGMQHDX3Bq6", + "object": "chat.completion", + "created": 1791225875, + "model": "gpt-5.6-sol", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello!", "refusal": None, "annotations": []}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 19, + "completion_tokens": 5, + "total_tokens": 24, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "audio_tokens": 0}, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, + expected_litellm_response={ + "id": "chatcmpl-EVhxr75iujjmwE0F2bgGMQHDX3Bq6", + "created": 1791225875, + "model": "openai/gpt-5.6-sol", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello!", + "role": "assistant", + "refusal": None, + "annotations": [], + }, + "provider_specific_fields": {}, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, +) + + +GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "openai/gpt-5.6-luna", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/chat/completions", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.6-luna", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "id": "chatcmpl-EVhxvZldBlNE3r8WZ8qJkxC92PmAr", + "object": "chat.completion", + "created": 1791225879, + "model": "gpt-5.6-luna", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello!", "refusal": None, "annotations": []}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 19, + "completion_tokens": 5, + "total_tokens": 24, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "audio_tokens": 0}, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, + expected_litellm_response={ + "id": "chatcmpl-EVhxvZldBlNE3r8WZ8qJkxC92PmAr", + "created": 1791225879, + "model": "openai/gpt-5.6-luna", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello!", + "role": "assistant", + "refusal": None, + "annotations": [], + }, + "provider_specific_fields": {}, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, +) + + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "openai/gpt-6-luna", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/chat/completions", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-6-luna", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "id": "chatcmpl-EVhxzlhJ6vfxv0dDB3QzTZvHJWTno", + "object": "chat.completion", + "created": 1791225883, + "model": "gpt-6-luna", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello!", "refusal": None, "annotations": []}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 19, + "completion_tokens": 5, + "total_tokens": 24, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "audio_tokens": 0}, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, + expected_litellm_response={ + "id": "chatcmpl-EVhxzlhJ6vfxv0dDB3QzTZvHJWTno", + "created": 1791225883, + "model": "openai/gpt-6-luna", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello!", + "role": "assistant", + "refusal": None, + "annotations": [], + }, + "provider_specific_fields": {}, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, +) + + +GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "openai/gpt-6.1-sol", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/chat/completions", + expected_provider_headers={ + "content-type": "application/json", + "authorization": "Bearer synthetic-openai-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-6.1-sol", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "id": "chatcmpl-EVhy2Sc0BN5sjJZK2Wa6l3CtsQ2ot", + "object": "chat.completion", + "created": 1791225886, + "model": "gpt-6.1-sol", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello!", "refusal": None, "annotations": []}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 19, + "completion_tokens": 5, + "total_tokens": 24, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "audio_tokens": 0}, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, + expected_litellm_response={ + "id": "chatcmpl-EVhy2Sc0BN5sjJZK2Wa6l3CtsQ2ot", + "created": 1791225886, + "model": "openai/gpt-6.1-sol", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello!", + "role": "assistant", + "refusal": None, + "annotations": [], + }, + "provider_specific_fields": {}, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + }, + "service_tier": "default", + "system_fingerprint": None, + }, +) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py new file mode 100644 index 00000000000..9a79456892c --- /dev/null +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_openai.py @@ -0,0 +1,30 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.bases.openai import ( + GPT_5_4_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + GPT_6_LUNA_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + GPT_5_4_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_6_LUNA_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_chat_completions_basic_openai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + pytest.skip( + "BUG: LIT-9235 chat completions moves message.refusal into provider_specific_fields and drops null system_fingerprint and logprobs" + ) + run(case, gateway, provider) diff --git a/tests/integration/translation/messages/bases/openai.py b/tests/integration/translation/messages/bases/openai.py new file mode 100644 index 00000000000..a027ae27bd2 --- /dev/null +++ b/tests/integration/translation/messages/bases/openai.py @@ -0,0 +1,483 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + + +GPT_5_4_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "openai/gpt-5.4", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-5.4", + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], + "include": ["reasoning.encrypted_content"], + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_0d51005efcef0683006ac3f00d0a3087d098c079bde6c6ee3c", + "object": "response", + "created_at": 1791225869, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225869, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.4-2026-03-05", + "moderation": None, + "output": [ + { + "id": "msg_0d51005efcef0683006ac3f00d8dc487d0aac89eb25a71a11f", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "current_turn", "effort": "none", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo3NGI3ZDQzZjBmNjQ1MjlkYWY4NWQ4MjRmNjE3ZGYzNDRjZTdhNGI3NjI3YTYzMDMzZGUzMDVmNTE2NTRlZjc2O3Jlc3BvbnNlX2lkOnJlc3BfMGQ1MTAwNWVmY2VmMDY4MzAwNmFjM2YwMGQwYTMwODdkMDk4YzA3OWJkZTZjNmVlM2M=", + "type": "message", + "role": "assistant", + "model": "openai/gpt-5.4", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + + +GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "openai/gpt-5.6-sol", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-5.6-sol", + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], + "include": ["reasoning.encrypted_content"], + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_006b1d3db27ce558006ac3f0119c5887d0b0fda575ab760d9d", + "object": "response", + "created_at": 1791225873, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225874, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.6-sol", + "moderation": None, + "output": [ + { + "id": "msg_006b1d3db27ce558006ac3f012169487d0b3574a945281b342", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDplNDZkYjA4Yjk5OGY1YmZmMTFhYTViNWIxMjJkMGQxZGVhZDM5MzI2MWZlZDU2OTc3M2FmOTNlMzBlYTMzNTc4O3Jlc3BvbnNlX2lkOnJlc3BfMDA2YjFkM2RiMjdjZTU1ODAwNmFjM2YwMTE5YzU4ODdkMGIwZmRhNTc1YWI3NjBkOWQ=", + "type": "message", + "role": "assistant", + "model": "openai/gpt-5.6-sol", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + + +GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "openai/gpt-5.6-luna", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-5.6-luna", + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], + "include": ["reasoning.encrypted_content"], + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_0fc0133ee7ba1828006ac3f015182c87d0bbc9b785a3a5a335", + "object": "response", + "created_at": 1791225877, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225877, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.6-luna", + "moderation": None, + "output": [ + { + "id": "msg_0fc0133ee7ba1828006ac3f015b22887d090ad19f7bfc86fd3", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo1MGY4ZWQzY2Q2ZWQ0NWFlMGQ3YWYxODI5YTJiNzg5M2I3ZWIxMjA5MTQ2Mzk5MTMyMGMwODFlNzE1NTc2OTM2O3Jlc3BvbnNlX2lkOnJlc3BfMGZjMDEzM2VlN2JhMTgyODAwNmFjM2YwMTUxODJjODdkMGJiYzliNzg1YTNhNWEzMzU=", + "type": "message", + "role": "assistant", + "model": "openai/gpt-5.6-luna", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "openai/gpt-6-luna", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-6-luna", + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], + "include": ["reasoning.encrypted_content"], + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_0dda38229cbb1547006ac3f0195a8487d0818613a635a9e0e2", + "object": "response", + "created_at": 1791225881, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225882, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-6-luna", + "moderation": None, + "output": [ + { + "id": "msg_0dda38229cbb1547006ac3f019f55887d08644c44c51109378", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo0ZmY4OWU3OTgyMGFkZDVjMmM1ZmZlM2MyODk4ZDNkOGEzZDczY2IxY2ExN2JmZGMxYTRmNTRlZmFlNzhkMTIzO3Jlc3BvbnNlX2lkOnJlc3BfMGRkYTM4MjI5Y2JiMTU0NzAwNmFjM2YwMTk1YTg0ODdkMDgxODYxM2E2MzVhOWUwZTI=", + "type": "message", + "role": "assistant", + "model": "openai/gpt-6-luna", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + + +GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "openai/gpt-6.1-sol", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-6.1-sol", + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}], + "include": ["reasoning.encrypted_content"], + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_03474a48789eb520006ac3f01d28ec87d092a151bed4f1cde6", + "object": "response", + "created_at": 1791225885, + "status": "completed", + "access_programs": None, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225886, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-6.1-sol", + "moderation": None, + "output": [ + { + "id": "msg_03474a48789eb520006ac3f01e31e887d0a7384c283c8d0934", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDphMTYxN2NmYzNlODRjNzYxNDdjNTgxMmFhMjIwNGU0MDczOTM0M2YyMTg4YzQxNzIxMzRlOTc1ZTQ0YzQ3MjgyO3Jlc3BvbnNlX2lkOnJlc3BfMDM0NzRhNDg3ODllYjUyMDAwNmFjM2YwMWQyOGVjODdkMDkyYTE1MWJlZDRmMWNkZTY=", + "type": "message", + "role": "assistant", + "model": "openai/gpt-6.1-sol", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 6}, + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_openai.py b/tests/integration/translation/messages/basic/test_messages_basic_openai.py new file mode 100644 index 00000000000..973bfd78d29 --- /dev/null +++ b/tests/integration/translation/messages/basic/test_messages_basic_openai.py @@ -0,0 +1,27 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.openai import ( + GPT_5_4_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + GPT_6_LUNA_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + GPT_5_4_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_6_LUNA_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_messages_basic_openai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + run(case, gateway, provider) diff --git a/tests/integration/translation/responses/bases/openai.py b/tests/integration/translation/responses/bases/openai.py new file mode 100644 index 00000000000..dae053c9de0 --- /dev/null +++ b/tests/integration/translation/responses/bases/openai.py @@ -0,0 +1,784 @@ +from typing import Final +from unittest.mock import ANY + +from integration.translation.case import TranslationTestCase + + +GPT_5_4_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "openai/gpt-5.4", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-5.4", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_09266ad9f58a7706006ac3f0106a2c87d09155b2c4aed60612", + "object": "response", + "created_at": 1791225872, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225873, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.4-2026-03-05", + "moderation": None, + "output": [ + { + "id": "msg_09266ad9f58a7706006ac3f0111c0487d0834c20939578eaca", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "current_turn", "effort": "none", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791225872, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "openai/gpt-5.4", + "object": "response", + "output": [ + { + "id": "msg_09266ad9f58a7706006ac3f0111c0487d0834c20939578eaca", + "content": [{"annotations": [], "text": "Hello.", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "current_turn", "effort": "none", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225873, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + + +GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "openai/gpt-5.6-sol", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-5.6-sol", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_0c9879957c1ae38c006ac3f013f1d487d0addcbd528bf79ce8", + "object": "response", + "created_at": 1791225875, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225876, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.6-sol", + "moderation": None, + "output": [ + { + "id": "msg_0c9879957c1ae38c006ac3f0146a0087d0a02d07eeff59b3c6", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791225875, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "openai/gpt-5.6-sol", + "object": "response", + "output": [ + { + "id": "msg_0c9879957c1ae38c006ac3f0146a0087d0a02d07eeff59b3c6", + "content": [{"annotations": [], "text": "Hello.", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225876, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + + +GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "openai/gpt-5.6-luna", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-5.6-luna", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_06fba2eaed98666c006ac3f0181d8c87d09e79c4255c5a24c4", + "object": "response", + "created_at": 1791225880, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225880, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.6-luna", + "moderation": None, + "output": [ + { + "id": "msg_06fba2eaed98666c006ac3f018dd9487d08c7e921025100a6d", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791225880, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "openai/gpt-5.6-luna", + "object": "response", + "output": [ + { + "id": "msg_06fba2eaed98666c006ac3f018dd9487d08c7e921025100a6d", + "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225880, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "openai/gpt-6-luna", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-6-luna", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_05a79fb530391552006ac3f01c4cec87d0881e61192e1046ba", + "object": "response", + "created_at": 1791225884, + "status": "completed", + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225884, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-6-luna", + "moderation": None, + "output": [ + { + "id": "msg_05a79fb530391552006ac3f01cb60c87d082152e269d863aa4", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791225884, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "openai/gpt-6-luna", + "object": "response", + "output": [ + { + "id": "msg_05a79fb530391552006ac3f01cb60c87d082152e269d863aa4", + "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "access_programs": {"cyber": "daybreak_blue"}, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225884, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + + +GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "openai/gpt-6.1-sol", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/responses", + expected_provider_headers={"content-type": "application/json", "authorization": "Bearer synthetic-openai-key"}, + expected_provider_request={ + "model": "gpt-6.1-sol", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_05eee41d540f8798006ac3f01fff8887d0bf3cdc6e114644ae", + "object": "response", + "created_at": 1791225888, + "status": "completed", + "access_programs": None, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225889, + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-6.1-sol", + "moderation": None, + "output": [ + { + "id": "msg_05eee41d540f8798006ac3f02102c887d08143f5010642dc50", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791225888, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "openai/gpt-6.1-sol", + "object": "response", + "output": [ + { + "id": "msg_05eee41d540f8798006ac3f02102c887d08143f5010642dc50", + "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "access_programs": None, + "background": False, + "billing": {"payer": "developer"}, + "completed_at": 1791225889, + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_openai.py b/tests/integration/translation/responses/basic/test_responses_basic_openai.py new file mode 100644 index 00000000000..4cd4e2e230f --- /dev/null +++ b/tests/integration/translation/responses/basic/test_responses_basic_openai.py @@ -0,0 +1,27 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.responses.bases.openai import ( + GPT_5_4_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + GPT_6_LUNA_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + GPT_5_4_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_6_LUNA_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_responses_basic_openai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + run(case, gateway, provider)