diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 656733ff41f..09b1ea4c772 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -105,6 +105,36 @@ model_list: model: openai/responses/gpt-6.1-sol api_base: http://127.0.0.1:8191 api_key: synthetic-openai-key + - model_name: azure/gpt-5.4 + litellm_params: + model: azure/gpt-5.4 + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-key + api_version: "2025-04-01-preview" + - model_name: azure/gpt-5.6-sol + litellm_params: + model: azure/gpt-5.6-sol + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-key + api_version: "2025-04-01-preview" + - model_name: azure/gpt-5.6-luna + litellm_params: + model: azure/gpt-5.6-luna + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-key + api_version: "2025-04-01-preview" + - model_name: azure/gpt-6-luna + litellm_params: + model: azure/gpt-6-luna + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-key + api_version: "2025-04-01-preview" + - model_name: azure/gpt-6.1-sol + litellm_params: + model: azure/gpt-6.1-sol + api_base: http://127.0.0.1:8191 + api_key: synthetic-azure-key + api_version: "2025-04-01-preview" general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL diff --git a/tests/integration/translation/chat_completions/bases/azure.py b/tests/integration/translation/chat_completions/bases/azure.py new file mode 100644 index 00000000000..523cf982a50 --- /dev/null +++ b/tests/integration/translation/chat_completions/bases/azure.py @@ -0,0 +1,768 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +GPT_5_4_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "azure/gpt-5.4", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-5.4/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.4", + "max_completion_tokens": 64, + "stream": False, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello.", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231844, + "id": "chatcmpl-EVjW8mtOvQmVYDtEuv7KNgydJESBC", + "model": "gpt-5.4-2026-03-05", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260919103649-bf00bd58-default-r2-dp0-default"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 6, + "engine_ttft_ms": 134, + "engine_ttlt_ms": 168, + "pre_inference_ms": 74, + "service_tbt_ms": 6, + "service_ttft_ms": 564, + "service_ttlt_ms": 592, + "user_visible_ttft_ms": 490, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjW8mtOvQmVYDtEuv7KNgydJESBC", + "created": 1791231844, + "model": "azure/gpt-5.4", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "Hello.", "role": "assistant", "annotations": [], "refusal": None}, + "provider_specific_fields": { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + } + }, + "logprobs": None, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, + "latency_checkpoint": { + "engine_tbt_ms": 6, + "engine_ttft_ms": 134, + "engine_ttlt_ms": 168, + "pre_inference_ms": 74, + "service_tbt_ms": 6, + "service_ttft_ms": 564, + "service_ttlt_ms": 592, + "user_visible_ttft_ms": 490, + }, + }, + "service_tier": "default", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260919103649-bf00bd58-default-r2-dp0-default"}, + "system_fingerprint": None, + }, +) + +GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "azure/gpt-5.6-sol", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-5.6-sol/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.6-sol", + "max_completion_tokens": 64, + "stream": False, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello.", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231846, + "id": "chatcmpl-EVjWAtsNezzFIBBSE6qy0TnpRv0Ll", + "model": "gpt-5.6-sol-2026-07-09", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260930154032-380c3f99-prefill-r9-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 7, + "engine_ttft_ms": 228, + "engine_ttlt_ms": 269, + "pre_inference_ms": 78, + "service_tbt_ms": 8, + "service_ttft_ms": 348, + "service_ttlt_ms": 387, + "user_visible_ttft_ms": 270, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWAtsNezzFIBBSE6qy0TnpRv0Ll", + "created": 1791231846, + "model": "azure/gpt-5.6-sol", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "Hello.", "role": "assistant", "annotations": [], "refusal": None}, + "provider_specific_fields": { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + } + }, + "logprobs": None, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 7, + "engine_ttft_ms": 228, + "engine_ttlt_ms": 269, + "pre_inference_ms": 78, + "service_tbt_ms": 8, + "service_ttft_ms": 348, + "service_ttlt_ms": 387, + "user_visible_ttft_ms": 270, + }, + }, + "service_tier": "default", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260930154032-380c3f99-prefill-r9-dp0-prefill"}, + "system_fingerprint": None, + }, +) + +GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "azure/gpt-5.6-luna", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-5.6-luna/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.6-luna", + "max_completion_tokens": 64, + "stream": False, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231848, + "id": "chatcmpl-EVjWC8K4D1wYOwETMUzs0oFLnsvKg", + "model": "gpt-5.6-luna-2026-07-09", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260925052750-47dc37a0-prefill-r7-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 12, + "engine_ttft_ms": 58, + "engine_ttlt_ms": 130, + "pre_inference_ms": 70, + "service_tbt_ms": 13, + "service_ttft_ms": 224, + "service_ttlt_ms": 290, + "user_visible_ttft_ms": 154, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWC8K4D1wYOwETMUzs0oFLnsvKg", + "created": 1791231848, + "model": "azure/gpt-5.6-luna", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "Hello!", "role": "assistant", "annotations": [], "refusal": None}, + "provider_specific_fields": { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + } + }, + "logprobs": None, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 12, + "engine_ttft_ms": 58, + "engine_ttlt_ms": 130, + "pre_inference_ms": 70, + "service_tbt_ms": 13, + "service_ttft_ms": 224, + "service_ttlt_ms": 290, + "user_visible_ttft_ms": 154, + }, + }, + "service_tier": "default", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260925052750-47dc37a0-prefill-r7-dp0-prefill"}, + "system_fingerprint": None, + }, +) + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "azure/gpt-6-luna", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-6-luna/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-6-luna", + "max_completion_tokens": 64, + "stream": False, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231849, + "id": "chatcmpl-EVjWDVYMA8Y26Lw8Jk3617Q3kWSUS", + "model": "gpt-6-luna-2026-09-22", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260930053954-f91e8c7b-prefill-r7-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 2, + "engine_ttft_ms": 65, + "engine_ttlt_ms": 80, + "pre_inference_ms": 83, + "service_tbt_ms": 5, + "service_ttft_ms": 282, + "service_ttlt_ms": 306, + "user_visible_ttft_ms": 199, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWDVYMA8Y26Lw8Jk3617Q3kWSUS", + "created": 1791231849, + "model": "azure/gpt-6-luna", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "Hello!", "role": "assistant", "annotations": [], "refusal": None}, + "provider_specific_fields": { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + } + }, + "logprobs": None, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 2, + "engine_ttft_ms": 65, + "engine_ttlt_ms": 80, + "pre_inference_ms": 83, + "service_tbt_ms": 5, + "service_ttft_ms": 282, + "service_ttlt_ms": 306, + "user_visible_ttft_ms": 199, + }, + }, + "service_tier": "default", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260930053954-f91e8c7b-prefill-r7-dp0-prefill"}, + "system_fingerprint": None, + }, +) + +GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/chat/completions", + litellm_request={ + "model": "azure/gpt-6.1-sol", + "max_tokens": 64, + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-6.1-sol/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-6.1-sol", + "max_completion_tokens": 64, + "stream": False, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231851, + "id": "chatcmpl-EVjWFBfdrUdRu3rbi6eAXbuO4zfNm", + "model": "gpt-6.1-sol-2026-09-29", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260929014647-f3a90410-prefill-r11-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 8, + "engine_ttft_ms": 440, + "engine_ttlt_ms": 491, + "pre_inference_ms": 70, + "service_tbt_ms": 9, + "service_ttft_ms": 773, + "service_ttlt_ms": 819, + "user_visible_ttft_ms": 703, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWFBfdrUdRu3rbi6eAXbuO4zfNm", + "created": 1791231851, + "model": "azure/gpt-6.1-sol", + "object": "chat.completion", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "Hello!", "role": "assistant", "annotations": [], "refusal": None}, + "provider_specific_fields": { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + } + }, + "logprobs": None, + } + ], + "usage": { + "completion_tokens": 5, + "prompt_tokens": 19, + "total_tokens": 24, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 8, + "engine_ttft_ms": 440, + "engine_ttlt_ms": 491, + "pre_inference_ms": 70, + "service_tbt_ms": 9, + "service_ttft_ms": 773, + "service_ttlt_ms": 819, + "user_visible_ttft_ms": 703, + }, + }, + "service_tier": "default", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260929014647-f3a90410-prefill-r11-dp0-prefill"}, + "system_fingerprint": None, + }, +) diff --git a/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py new file mode 100644 index 00000000000..ffdf13cdd2b --- /dev/null +++ b/tests/integration/translation/chat_completions/basic/test_chat_completions_basic_azure.py @@ -0,0 +1,30 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.bases.azure import ( + GPT_5_4_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + GPT_6_LUNA_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + GPT_5_4_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_6_LUNA_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_chat_completions_basic_azure(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + pytest.skip( + "BUG: LIT-9235 chat completions moves message.refusal into provider_specific_fields and drops null system_fingerprint and logprobs" + ) + run(case, gateway, provider) diff --git a/tests/integration/translation/messages/bases/azure.py b/tests/integration/translation/messages/bases/azure.py new file mode 100644 index 00000000000..6f0b924edae --- /dev/null +++ b/tests/integration/translation/messages/bases/azure.py @@ -0,0 +1,483 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +GPT_5_4_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "azure/gpt-5.4", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-5.4/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.4", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello.", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231864, + "id": "chatcmpl-EVjWSpHSltjjRtiCWAwJpzcB7dyOQ", + "model": "gpt-5.4-2026-03-05", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260919103649-bf00bd58-default-r2-dp0-default"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 11, + "engine_ttft_ms": 84, + "engine_ttlt_ms": 152, + "pre_inference_ms": 63, + "service_tbt_ms": 12, + "service_ttft_ms": 516, + "service_ttlt_ms": 578, + "user_visible_ttft_ms": 453, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWSpHSltjjRtiCWAwJpzcB7dyOQ", + "type": "message", + "role": "assistant", + "model": "azure/gpt-5.4", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 5}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "azure/gpt-5.6-sol", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-5.6-sol/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.6-sol", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello.", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231865, + "id": "chatcmpl-EVjWTALQyI0QnRBpbPzjE9NvDPIJG", + "model": "gpt-5.6-sol-2026-07-09", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260930154032-380c3f99-prefill-r9-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 7, + "engine_ttft_ms": 144, + "engine_ttlt_ms": 186, + "pre_inference_ms": 66, + "service_tbt_ms": 4, + "service_ttft_ms": 267, + "service_ttlt_ms": 286, + "user_visible_ttft_ms": 201, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWTALQyI0QnRBpbPzjE9NvDPIJG", + "type": "message", + "role": "assistant", + "model": "azure/gpt-5.6-sol", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 5}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "azure/gpt-5.6-luna", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-5.6-luna/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-5.6-luna", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231867, + "id": "chatcmpl-EVjWVHRXX2btCuzlg1wSnpqkpZFQZ", + "model": "gpt-5.6-luna-2026-07-09", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260925052750-47dc37a0-prefill-r12-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 12, + "engine_ttft_ms": 64, + "engine_ttlt_ms": 138, + "pre_inference_ms": 88, + "service_tbt_ms": 14, + "service_ttft_ms": 248, + "service_ttlt_ms": 317, + "user_visible_ttft_ms": 159, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWVHRXX2btCuzlg1wSnpqkpZFQZ", + "type": "message", + "role": "assistant", + "model": "azure/gpt-5.6-luna", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 5}, + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "azure/gpt-6-luna", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-6-luna/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-6-luna", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231868, + "id": "chatcmpl-EVjWWeBeTmIBg80iLiSeFNeYYY20z", + "model": "gpt-6-luna-2026-09-22", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260930053954-f91e8c7b-prefill-r6-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 3, + "engine_ttft_ms": 63, + "engine_ttlt_ms": 79, + "pre_inference_ms": 81, + "service_tbt_ms": 4, + "service_ttft_ms": 280, + "service_ttlt_ms": 301, + "user_visible_ttft_ms": 199, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWWeBeTmIBg80iLiSeFNeYYY20z", + "type": "message", + "role": "assistant", + "model": "azure/gpt-6-luna", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 5}, + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) + +GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "azure/gpt-6.1-sol", + "max_tokens": 64, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/deployments/gpt-6.1-sol/chat/completions?api-version=2025-04-01-preview", + expected_provider_headers={ + "content-type": "application/json", + "api-key": "synthetic-azure-key", + "authorization": "Bearer synthetic-azure-key", + }, + expected_provider_request={ + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "model": "gpt-6.1-sol", + "max_completion_tokens": 64, + }, + mock_provider_response={ + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_code": {"detected": False, "filtered": False}, + "protected_material_text": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": {"annotations": [], "content": "Hello!", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791231869, + "id": "chatcmpl-EVjWXoW5I8J7jZD6gsZ3ONxzOqZ27", + "model": "gpt-6.1-sol-2026-09-29", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "routing": {"serving_pipereplica": "d20260929014647-f3a90410-prefill-r11-dp0-prefill"}, + "service_tier": "default", + "system_fingerprint": None, + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 9, + "engine_ttft_ms": 477, + "engine_ttlt_ms": 530, + "pre_inference_ms": 44, + "service_tbt_ms": 8, + "service_ttft_ms": 795, + "service_ttlt_ms": 833, + "user_visible_ttft_ms": 750, + }, + "prompt_tokens": 19, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 24, + }, + }, + expected_litellm_response={ + "id": "chatcmpl-EVjWXoW5I8J7jZD6gsZ3ONxzOqZ27", + "type": "message", + "role": "assistant", + "model": "azure/gpt-6.1-sol", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 5}, + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_azure.py b/tests/integration/translation/messages/basic/test_messages_basic_azure.py new file mode 100644 index 00000000000..dfde788555f --- /dev/null +++ b/tests/integration/translation/messages/basic/test_messages_basic_azure.py @@ -0,0 +1,27 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.azure import ( + GPT_5_4_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + GPT_6_LUNA_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + GPT_5_4_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_6_LUNA_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_messages_basic_azure(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + run(case, gateway, provider) diff --git a/tests/integration/translation/responses/bases/azure.py b/tests/integration/translation/responses/bases/azure.py new file mode 100644 index 00000000000..458a66abb52 --- /dev/null +++ b/tests/integration/translation/responses/bases/azure.py @@ -0,0 +1,1049 @@ +from typing import Final +from unittest.mock import ANY + +from integration.translation.case import TranslationTestCase + +GPT_5_4_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "azure/gpt-5.4", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={ + "model": "gpt-5.4", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_0762039bed856005006ac4076cdb5c8194a48664a1b2ef25ae", + "object": "response", + "created_at": 1791231852, + "status": "completed", + "background": False, + "completed_at": 1791231853, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 842, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.4", + "moderation": None, + "output": [ + { + "id": "msg_0762039bed856005006ac4076d77f08194b260c944a432ba1a", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "in_memory", + "reasoning": {"context": "current_turn", "effort": "none", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791231852, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "azure/gpt-5.4", + "object": "response", + "output": [ + { + "id": "msg_0762039bed856005006ac4076d77f08194b260c944a432ba1a", + "content": [{"annotations": [], "text": "Hello.", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "current_turn", "effort": "none", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "background": False, + "completed_at": 1791231853, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 842, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "in_memory", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + +GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "azure/gpt-5.6-sol", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={ + "model": "gpt-5.6-sol", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_08c09f398a5c1990006ac4076e8c5c819498395510175e9c98", + "object": "response", + "created_at": 1791231854, + "status": "completed", + "background": False, + "completed_at": 1791231857, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.6-sol", + "moderation": None, + "output": [ + { + "id": "msg_08c09f398a5c1990006ac4077114e48194a52f30a20676db99", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791231854, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "azure/gpt-5.6-sol", + "object": "response", + "output": [ + { + "id": "msg_08c09f398a5c1990006ac4077114e48194a52f30a20676db99", + "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "background": False, + "completed_at": 1791231857, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + +GPT_5_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "azure/gpt-5.6-luna", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={ + "model": "gpt-5.6-luna", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_00c2ebc008a733e5006ac40772599081938a97e3688d1ff7f9", + "object": "response", + "created_at": 1791231858, + "status": "completed", + "background": False, + "completed_at": 1791231859, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-5.6-luna", + "moderation": None, + "output": [ + { + "id": "msg_00c2ebc008a733e5006ac40772ae908193b01d2c647918c8d4", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791231858, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "azure/gpt-5.6-luna", + "object": "response", + "output": [ + { + "id": "msg_00c2ebc008a733e5006ac40772ae908193b01d2c647918c8d4", + "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "background": False, + "completed_at": 1791231859, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + +GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "azure/gpt-6-luna", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={ + "model": "gpt-6-luna", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_0f6bf1d4fb9f1e26006ac4077413a08193b9ef8007e2a67444", + "object": "response", + "created_at": 1791231860, + "status": "completed", + "background": False, + "completed_at": 1791231860, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-6-luna", + "moderation": None, + "output": [ + { + "id": "msg_0f6bf1d4fb9f1e26006ac4077472cc8193a8c6b9fd6d98c8be", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791231860, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "azure/gpt-6-luna", + "object": "response", + "output": [ + { + "id": "msg_0f6bf1d4fb9f1e26006ac4077472cc8193a8c6b9fd6d98c8be", + "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "background": False, + "completed_at": 1791231860, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) + +GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/responses", + litellm_request={ + "model": "azure/gpt-6.1-sol", + "max_output_tokens": 64, + "instructions": "You are a terse assistant.", + "input": "Say hello.", + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/responses?api-version=2025-04-01-preview", + expected_provider_headers={"content-type": "application/json", "api-key": "synthetic-azure-key"}, + expected_provider_request={ + "model": "gpt-6.1-sol", + "input": "Say hello.", + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + }, + mock_provider_response={ + "id": "resp_0da7f0a44a7da608006ac40775cfbc8197991a6f98f5111832", + "object": "response", + "created_at": 1791231861, + "status": "completed", + "background": False, + "completed_at": 1791231863, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "error": None, + "frequency_penalty": 0.0, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "max_output_tokens": 64, + "max_tool_calls": None, + "model": "gpt-6.1-sol", + "moderation": None, + "output": [ + { + "id": "msg_0da7f0a44a7da608006ac4077719f48197830b80e5759a5d75", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello!"}], + "phase": "final_answer", + "role": "assistant", + } + ], + "parallel_tool_calls": True, + "presence_penalty": 0.0, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 6, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, + }, + "user": None, + "metadata": {}, + }, + expected_litellm_response={ + "id": ANY, + "created_at": 1791231861, + "error": None, + "incomplete_details": None, + "instructions": "You are a terse assistant.", + "metadata": {}, + "model": "azure/gpt-6.1-sol", + "object": "response", + "output": [ + { + "id": "msg_0da7f0a44a7da608006ac4077719f48197830b80e5759a5d75", + "content": [{"annotations": [], "text": "Hello!", "type": "output_text", "logprobs": []}], + "role": "assistant", + "status": "completed", + "type": "message", + "phase": "final_answer", + } + ], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "max_output_tokens": 64, + "previous_response_id": None, + "reasoning": {"context": "all_turns", "effort": "medium", "mode": "standard", "summary": None}, + "status": "completed", + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "truncation": "disabled", + "usage": { + "input_tokens": 19, + "input_tokens_details": { + "audio_tokens": None, + "cached_tokens": 0, + "cached_tokens_details": None, + "image_tokens": None, + "text_tokens": None, + "video_tokens": None, + "cache_write_tokens": 0, + }, + "output_tokens": 6, + "output_tokens_details": {"audio_tokens": None, "reasoning_tokens": 0, "text_tokens": None}, + "total_tokens": 25, + "cost": None, + }, + "user": None, + "store": True, + "background": False, + "completed_at": 1791231863, + "content_filters": [ + { + "blocked": False, + "source_type": "prompt", + "content_filter_raw": [], + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "jailbreak": {"detected": False, "filtered": False}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 788, "check_offset": 0}, + }, + { + "blocked": False, + "source_type": "completion", + "content_filter_raw": [], + "content_filter_results": { + "protected_material_text": {"detected": False, "filtered": False}, + "protected_material_code": {"detected": False, "filtered": False}, + "hate": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + }, + "content_filter_offsets": {"start_offset": 0, "end_offset": 6, "check_offset": 0}, + }, + ], + "frequency_penalty": 0.0, + "max_tool_calls": None, + "moderation": None, + "presence_penalty": 0.0, + "prompt_cache_key": None, + "prompt_cache_retention": "24h", + "safety_identifier": None, + "service_tier": "default", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "total_tokens": 0, + }, + "web_search": {"num_requests": 0}, + }, + "top_logprobs": 0, + }, +) diff --git a/tests/integration/translation/responses/basic/test_responses_basic_azure.py b/tests/integration/translation/responses/basic/test_responses_basic_azure.py new file mode 100644 index 00000000000..dee63e664c9 --- /dev/null +++ b/tests/integration/translation/responses/basic/test_responses_basic_azure.py @@ -0,0 +1,27 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.responses.bases.azure import ( + GPT_5_4_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + GPT_6_LUNA_TEST_CASE, +) +from integration.translation.runner import run + + +@pytest.mark.parametrize( + "case", + [ + GPT_5_4_TEST_CASE, + GPT_5_6_SOL_TEST_CASE, + GPT_5_6_LUNA_TEST_CASE, + GPT_6_LUNA_TEST_CASE, + GPT_6_1_SOL_TEST_CASE, + ], + ids=lambda case: case.id, +) +def test_responses_basic_azure(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + run(case, gateway, provider) diff --git a/tests/integration/translation/runner.py b/tests/integration/translation/runner.py index 81c858ad303..1732bc69a92 100644 --- a/tests/integration/translation/runner.py +++ b/tests/integration/translation/runner.py @@ -15,7 +15,11 @@ def run(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) - received: Final = provider.received() assert [(request.method, request.target) for request in received] == [("POST", case.expected_provider_endpoint)] sent: Final = received[0] - assert {name: value for name, value in sent.headers.items() if name not in TRANSPORT_HEADERS} == dict(case.expected_provider_headers) + assert { + name: value + for name, value in sent.headers.items() + if name not in TRANSPORT_HEADERS and not name.startswith("x-stainless-") + } == dict(case.expected_provider_headers) assert json.loads(sent.body) == case.expected_provider_request assert response.status_code == case.expected_litellm_status_code, response.text assert response.json() == case.expected_litellm_response