mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
test(integration): basic translation cases for the openai_responses route (#44663)
* test(integration): openai_responses-route basic translation cases on messages, chat completions and responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): skip openai_responses-route chat completions basic cases until LIT-9196 is fixed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
d23a0636b4
commit
7123484c28
7 changed files with 818 additions and 0 deletions
|
|
@ -95,6 +95,16 @@ model_list:
|
|||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-bedrock-key
|
||||
aws_region_name: us-east-1
|
||||
- model_name: openai/responses/gpt-5.6-sol
|
||||
litellm_params:
|
||||
model: openai/responses/gpt-5.6-sol
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-openai-key
|
||||
- model_name: openai/responses/gpt-6.1-sol
|
||||
litellm_params:
|
||||
model: openai/responses/gpt-6.1-sol
|
||||
api_base: http://127.0.0.1:8191
|
||||
api_key: synthetic-openai-key
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
database_url: os.environ/DATABASE_URL
|
||||
|
|
|
|||
|
|
@ -0,0 +1,233 @@
|
|||
from typing import Final
|
||||
from unittest.mock import ANY
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/chat/completions",
|
||||
litellm_request={
|
||||
"model": "openai/responses/gpt-5.6-sol",
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a terse assistant."},
|
||||
{"role": "user", "content": "Say hello."},
|
||||
],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "gpt-5.6-sol",
|
||||
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}],
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"stream": False,
|
||||
},
|
||||
mock_provider_response={
|
||||
"id": "resp_0c11adb5b3fcca34006ac3f043a44087d0ba76b3b869764314",
|
||||
"object": "response",
|
||||
"created_at": 1791225923,
|
||||
"status": "completed",
|
||||
"access_programs": {"cyber": "daybreak_blue"},
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225924,
|
||||
"error": None,
|
||||
"frequency_penalty": 0.0,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"max_tool_calls": None,
|
||||
"model": "gpt-5.6-sol",
|
||||
"moderation": None,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_0c11adb5b3fcca34006ac3f0441f4c87d0bb9cf4f5c7dfbfd1",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"presence_penalty": 0.0,
|
||||
"previous_response_id": None,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"store": True,
|
||||
"temperature": 1.0,
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"tool_choice": "auto",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"tools": [],
|
||||
"top_logprobs": 0,
|
||||
"top_p": 0.98,
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 25,
|
||||
},
|
||||
"user": None,
|
||||
"metadata": {},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"id": "resp_0c11adb5b3fcca34006ac3f043a44087d0ba76b3b869764314",
|
||||
"created": ANY,
|
||||
"model": "openai/responses/gpt-5.6-sol",
|
||||
"object": "chat.completion",
|
||||
"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello!", "role": "assistant"}}],
|
||||
"usage": {
|
||||
"completion_tokens": 6,
|
||||
"prompt_tokens": 19,
|
||||
"total_tokens": 25,
|
||||
"completion_tokens_details": {"reasoning_tokens": 0},
|
||||
"prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0},
|
||||
},
|
||||
"access_programs": {"cyber": "daybreak_blue"},
|
||||
"billing": {"payer": "developer"},
|
||||
"frequency_penalty": 0.0,
|
||||
"presence_penalty": 0.0,
|
||||
"service_tier": "default",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/chat/completions",
|
||||
litellm_request={
|
||||
"model": "openai/responses/gpt-6.1-sol",
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a terse assistant."},
|
||||
{"role": "user", "content": "Say hello."},
|
||||
],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "gpt-6.1-sol",
|
||||
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}],
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"stream": False,
|
||||
},
|
||||
mock_provider_response={
|
||||
"id": "resp_0ff994dd79332d97006ac3f04a429487d09618de316e259d7f",
|
||||
"object": "response",
|
||||
"created_at": 1791225930,
|
||||
"status": "completed",
|
||||
"access_programs": None,
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225933,
|
||||
"error": None,
|
||||
"frequency_penalty": 0.0,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"max_tool_calls": None,
|
||||
"model": "gpt-6.1-sol",
|
||||
"moderation": None,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_0ff994dd79332d97006ac3f04cfcf487d088bec15ca90ae36b",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"presence_penalty": 0.0,
|
||||
"previous_response_id": None,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"store": True,
|
||||
"temperature": 1.0,
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"tool_choice": "auto",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"tools": [],
|
||||
"top_logprobs": 0,
|
||||
"top_p": 0.98,
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 25,
|
||||
},
|
||||
"user": None,
|
||||
"metadata": {},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"id": "resp_0ff994dd79332d97006ac3f04a429487d09618de316e259d7f",
|
||||
"created": ANY,
|
||||
"model": "openai/responses/gpt-6.1-sol",
|
||||
"object": "chat.completion",
|
||||
"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello!", "role": "assistant"}}],
|
||||
"usage": {
|
||||
"completion_tokens": 6,
|
||||
"prompt_tokens": 19,
|
||||
"total_tokens": 25,
|
||||
"completion_tokens_details": {"reasoning_tokens": 0},
|
||||
"prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0, "cache_creation_tokens": 0},
|
||||
},
|
||||
"billing": {"payer": "developer"},
|
||||
"frequency_penalty": 0.0,
|
||||
"presence_penalty": 0.0,
|
||||
"service_tier": "default",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.chat_completions.bases.openai_responses import (
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
)
|
||||
from integration.translation.runner import run
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
],
|
||||
ids=lambda case: case.id,
|
||||
)
|
||||
def test_chat_completions_basic_openai_responses(
|
||||
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
pytest.skip("BUG: LIT-9196 no-cache ignored on chat completions bridged to responses")
|
||||
run(case, gateway, provider)
|
||||
193
tests/integration/translation/messages/bases/openai_responses.py
Normal file
193
tests/integration/translation/messages/bases/openai_responses.py
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/messages",
|
||||
litellm_request={
|
||||
"model": "openai/responses/gpt-5.6-sol",
|
||||
"max_tokens": 64,
|
||||
"system": "You are a terse assistant.",
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "gpt-5.6-sol",
|
||||
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}],
|
||||
"include": ["reasoning.encrypted_content"],
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
},
|
||||
mock_provider_response={
|
||||
"id": "resp_0879800c1455ba3d006ac3f0427c4c87d0a632b130c8c464ea",
|
||||
"object": "response",
|
||||
"created_at": 1791225922,
|
||||
"status": "completed",
|
||||
"access_programs": {"cyber": "daybreak_blue"},
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225923,
|
||||
"error": None,
|
||||
"frequency_penalty": 0.0,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"max_tool_calls": None,
|
||||
"model": "gpt-5.6-sol",
|
||||
"moderation": None,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_0879800c1455ba3d006ac3f0430f2087d0bcdc00a8eba6a1ce",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"presence_penalty": 0.0,
|
||||
"previous_response_id": None,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"store": True,
|
||||
"temperature": 1.0,
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"tool_choice": "auto",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"tools": [],
|
||||
"top_logprobs": 0,
|
||||
"top_p": 0.98,
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 25,
|
||||
},
|
||||
"user": None,
|
||||
"metadata": {},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo1ZDQ1NmY1ODlhMDIwNzI4N2M0YzU2NmRhN2NkN2ViNjJlMzZmNGUwMTYxOTE4M2VhMzhiMzdlZGZhNDdlM2EwO3Jlc3BvbnNlX2lkOnJlc3BfMDg3OTgwMGMxNDU1YmEzZDAwNmFjM2YwNDI3YzRjODdkMGE2MzJiMTMwYzhjNDY0ZWE=",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "openai/responses/gpt-5.6-sol",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 19, "output_tokens": 6},
|
||||
"content": [{"type": "text", "text": "Hello!"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_details": None,
|
||||
},
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/messages",
|
||||
litellm_request={
|
||||
"model": "openai/responses/gpt-6.1-sol",
|
||||
"max_tokens": 64,
|
||||
"system": "You are a terse assistant.",
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "gpt-6.1-sol",
|
||||
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Say hello."}]}],
|
||||
"include": ["reasoning.encrypted_content"],
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
},
|
||||
mock_provider_response={
|
||||
"id": "resp_03446eafdb61cd31006ac3f046b20c87d0af12e5a0e55548fd",
|
||||
"object": "response",
|
||||
"created_at": 1791225926,
|
||||
"status": "completed",
|
||||
"access_programs": None,
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225928,
|
||||
"error": None,
|
||||
"frequency_penalty": 0.0,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"max_tool_calls": None,
|
||||
"model": "gpt-6.1-sol",
|
||||
"moderation": None,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_03446eafdb61cd31006ac3f048804087d0a22e0e94d0a6d50e",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"presence_penalty": 0.0,
|
||||
"previous_response_id": None,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"store": True,
|
||||
"temperature": 1.0,
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"tool_choice": "auto",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"tools": [],
|
||||
"top_logprobs": 0,
|
||||
"top_p": 0.98,
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 25,
|
||||
},
|
||||
"user": None,
|
||||
"metadata": {},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo3ZGE3NDY3Njc3N2IyZjVkYmQ3MDI1MTI1ZjEzMDdlYWQ0MjM3YTgzN2QxZTJmZDhkY2FkNjhlODM4ODU1Zjg2O3Jlc3BvbnNlX2lkOnJlc3BfMDM0NDZlYWZkYjYxY2QzMTAwNmFjM2YwNDZiMjBjODdkMGFmMTJlNWEwZTU1NTQ4ZmQ=",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "openai/responses/gpt-6.1-sol",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 19, "output_tokens": 6},
|
||||
"content": [{"type": "text", "text": "Hello!"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_details": None,
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.bases.openai_responses import (
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
)
|
||||
from integration.translation.runner import run
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
],
|
||||
ids=lambda case: case.id,
|
||||
)
|
||||
def test_messages_basic_openai_responses(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
run(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,314 @@
|
|||
from typing import Final
|
||||
from unittest.mock import ANY
|
||||
|
||||
from integration.translation.case import TranslationTestCase
|
||||
|
||||
GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/responses",
|
||||
litellm_request={
|
||||
"model": "openai/responses/gpt-5.6-sol",
|
||||
"max_output_tokens": 64,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"input": "Say hello.",
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "gpt-5.6-sol",
|
||||
"input": "Say hello.",
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
},
|
||||
mock_provider_response={
|
||||
"id": "resp_0691e5d81d357aec006ac3f045381487d0904424fd18d4adfb",
|
||||
"object": "response",
|
||||
"created_at": 1791225925,
|
||||
"status": "completed",
|
||||
"access_programs": {"cyber": "daybreak_blue"},
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225926,
|
||||
"error": None,
|
||||
"frequency_penalty": 0.0,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"max_tool_calls": None,
|
||||
"model": "gpt-5.6-sol",
|
||||
"moderation": None,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_0691e5d81d357aec006ac3f04611ac87d081b8e4e32967101d",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"presence_penalty": 0.0,
|
||||
"previous_response_id": None,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"store": True,
|
||||
"temperature": 1.0,
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"tool_choice": "auto",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"tools": [],
|
||||
"top_logprobs": 0,
|
||||
"top_p": 0.98,
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 25,
|
||||
},
|
||||
"user": None,
|
||||
"metadata": {},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"id": ANY,
|
||||
"created_at": 1791225925,
|
||||
"error": None,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"metadata": {},
|
||||
"model": "openai/responses/gpt-5.6-sol",
|
||||
"object": "response",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_0691e5d81d357aec006ac3f04611ac87d081b8e4e32967101d",
|
||||
"content": [{"annotations": [], "text": "Hello.", "type": "output_text", "logprobs": []}],
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"type": "message",
|
||||
"phase": "final_answer",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"temperature": 1.0,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"top_p": 0.98,
|
||||
"max_output_tokens": 64,
|
||||
"previous_response_id": None,
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"status": "completed",
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {
|
||||
"audio_tokens": None,
|
||||
"cached_tokens": 0,
|
||||
"cached_tokens_details": None,
|
||||
"image_tokens": None,
|
||||
"text_tokens": None,
|
||||
"video_tokens": None,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None},
|
||||
"total_tokens": 25,
|
||||
"cost": None,
|
||||
},
|
||||
"user": None,
|
||||
"store": True,
|
||||
"access_programs": {"cyber": "daybreak_blue"},
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225926,
|
||||
"frequency_penalty": 0.0,
|
||||
"max_tool_calls": None,
|
||||
"moderation": None,
|
||||
"presence_penalty": 0.0,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"top_logprobs": 0,
|
||||
},
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase(
|
||||
scenario="basic",
|
||||
litellm_endpoint="/v1/responses",
|
||||
litellm_request={
|
||||
"model": "openai/responses/gpt-6.1-sol",
|
||||
"max_output_tokens": 64,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"input": "Say hello.",
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
expected_provider_endpoint="/responses",
|
||||
expected_provider_headers={"authorization": "Bearer synthetic-openai-key", "content-type": "application/json"},
|
||||
expected_provider_request={
|
||||
"model": "gpt-6.1-sol",
|
||||
"input": "Say hello.",
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
},
|
||||
mock_provider_response={
|
||||
"id": "resp_069a914d6d8c0f52006ac3f04d744887d0a3f54d9df9741033",
|
||||
"object": "response",
|
||||
"created_at": 1791225933,
|
||||
"status": "completed",
|
||||
"access_programs": None,
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225934,
|
||||
"error": None,
|
||||
"frequency_penalty": 0.0,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"max_output_tokens": 64,
|
||||
"max_tool_calls": None,
|
||||
"model": "gpt-6.1-sol",
|
||||
"moderation": None,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_069a914d6d8c0f52006ac3f04e984087d095c875b7ae7b5542",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"presence_penalty": 0.0,
|
||||
"previous_response_id": None,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"store": True,
|
||||
"temperature": 1.0,
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"tool_choice": "auto",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"tools": [],
|
||||
"top_logprobs": 0,
|
||||
"top_p": 0.98,
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 25,
|
||||
},
|
||||
"user": None,
|
||||
"metadata": {},
|
||||
},
|
||||
expected_litellm_response={
|
||||
"id": ANY,
|
||||
"created_at": 1791225933,
|
||||
"error": None,
|
||||
"incomplete_details": None,
|
||||
"instructions": "You are a terse assistant.",
|
||||
"metadata": {},
|
||||
"model": "openai/responses/gpt-6.1-sol",
|
||||
"object": "response",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_069a914d6d8c0f52006ac3f04e984087d095c875b7ae7b5542",
|
||||
"content": [{"annotations": [], "text": "Hello.", "type": "output_text", "logprobs": []}],
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"type": "message",
|
||||
"phase": "final_answer",
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"temperature": 1.0,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"top_p": 0.98,
|
||||
"max_output_tokens": 64,
|
||||
"previous_response_id": None,
|
||||
"reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None},
|
||||
"status": "completed",
|
||||
"text": {"format": {"type": "text"}, "verbosity": "medium"},
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 19,
|
||||
"input_tokens_details": {
|
||||
"audio_tokens": None,
|
||||
"cached_tokens": 0,
|
||||
"cached_tokens_details": None,
|
||||
"image_tokens": None,
|
||||
"text_tokens": None,
|
||||
"video_tokens": None,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"output_tokens": 6,
|
||||
"output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None},
|
||||
"total_tokens": 25,
|
||||
"cost": None,
|
||||
},
|
||||
"user": None,
|
||||
"store": True,
|
||||
"access_programs": None,
|
||||
"background": False,
|
||||
"billing": {"payer": "developer"},
|
||||
"completed_at": 1791225934,
|
||||
"frequency_penalty": 0.0,
|
||||
"max_tool_calls": None,
|
||||
"moderation": None,
|
||||
"presence_penalty": 0.0,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": "24h",
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
"total_tokens": 0,
|
||||
},
|
||||
"web_search": {"num_requests": 0},
|
||||
},
|
||||
"top_logprobs": 0,
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.responses.bases.openai_responses import (
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
)
|
||||
from integration.translation.runner import run
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
GPT_5_6_SOL_TEST_CASE,
|
||||
GPT_6_1_SOL_TEST_CASE,
|
||||
],
|
||||
ids=lambda case: case.id,
|
||||
)
|
||||
def test_responses_basic_openai_responses(
|
||||
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
run(case, gateway, provider)
|
||||
Loading…
Add table
Reference in a new issue