diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 26280d35da0..4af90b08fce 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -9,6 +9,7 @@ - {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"} - {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"} - {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"} +- {id: llm.chat_completions.openai.drop_params.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: drop_params, streaming: nonstream, assertions: [works], source: "llms/openai/chat/gpt_5_transformation.py", rationale: "drop_params must drop temperature != 1 for gpt-5 models whose default reasoning_effort is active (gpt-5.5 defaults to medium), else the provider 400s (LIT-3797)", fail_before_fix: proven} - {id: llm.chat_completions.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route translated to Anthropic"} - {id: llm.chat_completions.anthropic.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming translation"} - {id: llm.chat_completions.anthropic.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude tool_use; high usage"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 1a6dc111e2b..300a6faf258 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -58,6 +58,7 @@ LlmCapability = Literal[ "assume_role", "basic", "count_tokens", + "drop_params", "long_context_1m", "mid_conversation_system", "pdf_input", diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 13992744f42..6eed58594a4 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -81,6 +81,54 @@ class TestChatCompletionsRegression: ), f"{model} ({route}): 200 with an empty completion (#28991): {response}" +class TestGpt5TemperatureDropParams: + """gpt-5 family models whose default reasoning_effort is active (gpt-5.5 + defaults to medium, unlike gpt-5.1 which defaults to none) reject + temperature != 1. A deployment with drop_params must drop the temperature + instead of forwarding it and surfacing the provider's 400 (LIT-3797).""" + + @pytest.mark.covers( + "llm.chat_completions.openai.drop_params.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_gpt55_drops_unsupported_temperature( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-gpt55-temp-drop-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model="openai/gpt-5.5", + api_key="os.environ/OPENAI_API_KEY", + drop_params=True, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + temperature=0.1, + max_tokens=512, + ), + ) + ) + assert response.choices, f"gpt-5.5 chat returned no choices: {response}" + message = response.choices[0].message + assert ( + message is not None and message.content and message.content.strip() + ), f"gpt-5.5 with temperature 0.1 returned an empty completion: {response}" + + class TestCohereChat: """Cohere via the OpenAI-compatible /chat/completions path.""" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 8b6c454fa1e..73697375553 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -172,6 +172,7 @@ class ChatBody(BaseModel): messages: list[ChatMessage] stream: bool = False max_tokens: int | None = None + temperature: float | None = None user: str | None = None metadata: ChatMetadata | None = None reasoning_effort: str | None = None @@ -568,6 +569,7 @@ class LiteLLMParamsBody(BaseModel): complexity_router_config: dict[str, object] | None = None mock_response: str | None = None timeout: float | None = None + drop_params: bool | None = None ModelMode = Literal["batch", "realtime", "image_generation"]