From 7075919584399edf431f23c572c8cfad4b45b549 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sat, 25 Jul 2026 10:32:08 -0700 Subject: [PATCH] test(e2e): point four suites at models the providers still serve (#34567) * test(e2e): point four suites at models the providers still serve Four llm_translation tests failed against upstream because the model they name no longer exists. Each replacement was verified against the live stage proxy. deepseek/deepseek-reasoner is gone; the DeepSeek API now lists only deepseek-v4-flash and deepseek-v4-pro. Use deepseek/deepseek-v4-pro, which still returns message.reasoning_content by default and still drops it for both reasoning_effort="none" and thinking={"type": "disabled"} (litellm maps the former to the latter, so the provider rejecting a bare "none" does not matter). amazon.titan-image-generator-v2:0 returns "This model version has reached the end of its life"; amazon.nova-canvas-v1:0 is the text-to-image model Bedrock still offers in us-east-1. Bedrock's Rerank API requires a full model ARN and rejects a bare model id with "The provided model ARN for reranking is invalid", regardless of model or region. Pass the ARN for cohere.rerank-v3-5:0, which is available in the stack's us-east-1. vertex_ai/gemini-embedding-2 404s as an unknown publisher model on this project; vertex_ai/text-embedding-005 returns a vector. * test(e2e): skip the hosted_vllm chat test when its server is unset test_hosted_vllm_chat_returns_content read os.environ["HOSTED_VLLM_API_BASE"] directly, so a stack without that env var failed the test with a bare KeyError instead of reporting an environment gap. The batches suite already skips on the same variable, and the vertex passthrough tests use pytest.skip for the same reason, so follow that idiom here. Drop the HOSTED_VLLM_API_KEY plumbing: the stage vLLM stand-in serves /v1/chat/completions unauthenticated, and api_key is optional on LiteLLMParamsBody, so passing it added nothing. Default the backend to the model that server actually serves, Qwen/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M, rather than a Llama id it never had. Verified against the live stage proxy: a deployment with just that model and api_base returns "hello". * fix(model_map): mark deepseek v4-pro and v4-flash as reasoning-capable Review on #34567 flagged that deepseek/deepseek-v4-pro is not marked reasoning-capable while the e2e control case requires reasoning_content back from it. The behavior premise is inverted, but it surfaced a real data gap: the model map never gained supports_reasoning for the v4 models when DeepSeek retired deepseek-reasoner, which did carry the flag. Both models do reason. Against the live API with no reasoning params, v4-pro returns 106 chars of reasoning_content and v4-flash returns 54, and both drop it for thinking={"type":"disabled"}. The stale flag had a real consequence beyond metadata: DeepSeekChatConfig ._thinking_mode_active() gates on supports_reasoning(), so with the flag unset it returned False even when a caller passed thinking={"type": "enabled"}, skipping the multi-turn check that reasoning_content be passed back on assistant messages. Param support itself was never gated, which is why reasoning_effort="none" still mapped to thinking disabled. Verified with LITELLM_LOCAL_MODEL_COST_MAP=True: supports_reasoning now reports True for deepseek/deepseek-v4-pro and deepseek/deepseek-v4-flash. tencent/deepseek-v4-pro is left alone; that route was not exercised here. --- litellm/model_prices_and_context_window_backup.json | 4 ++++ model_prices_and_context_window.json | 4 ++++ .../test_chat_completions_regression_e2e.py | 11 +++++++---- .../llm_translation/test_deepseek_reasoning_e2e.py | 6 +++--- .../llm_translation/test_embeddings_endpoint_e2e.py | 2 +- .../e2e/llm_translation/test_image_generation_e2e.py | 2 +- tests/e2e/llm_translation/test_rerank_e2e.py | 2 +- 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d43eda39b1f..f577406fc68 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45890,6 +45890,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -45915,6 +45916,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -45940,6 +45942,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -45999,6 +46002,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 749b2566c2a..af69d58f00a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46012,6 +46012,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -46037,6 +46038,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -46062,6 +46064,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -46087,6 +46090,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index af0e782e224..35be2254273 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -343,10 +343,14 @@ class TestHostedVllmChat: def test_hosted_vllm_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: - api_base = os.environ["HOSTED_VLLM_API_BASE"] - api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None + api_base = os.environ.get("HOSTED_VLLM_API_BASE") + if api_base is None: + pytest.skip( + "set HOSTED_VLLM_API_BASE (the live vLLM server this deployment targets)" + ) backend = ( - os.environ.get("HOSTED_VLLM_MODEL") or "meta-llama/Llama-3.2-3B-Instruct" + os.environ.get("HOSTED_VLLM_MODEL") + or "Qwen/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M" ).strip() model = f"e2e-vllm-chat-{unique_marker()}" model_id = client.proxy.create_model( @@ -354,7 +358,6 @@ class TestHostedVllmChat: LiteLLMParamsBody( model=f"hosted_vllm/{backend}", api_base=api_base, - api_key=api_key, ), ) resources.defer(lambda: client.proxy.delete_model(model_id)) diff --git a/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py b/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py index b06b241c0b5..8dfccf0d74b 100644 --- a/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py +++ b/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py @@ -5,7 +5,7 @@ DeepSeek's reasoner defaults thinking ON and surfaces the chain as ``reasoning_effort="none"`` and ``thinking={"type": "disabled"}``. The DeepSeek param mapper (``litellm/llms/deepseek/chat/transformation.py`` ``map_openai_params``) forwards both as ``thinking={"type": "disabled"}`` so the -outbound body carries a real disable signal and ``deepseek-reasoner`` returns no +outbound body carries a real disable signal and the reasoning model returns no ``reasoning_content``. This is the behavior tracked by LIT-3686 / GH #27453. The control case proves the model and path work (reasoning is returned when @@ -27,7 +27,7 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -REASONER = "deepseek/deepseek-reasoner" +REASONER = "deepseek/deepseek-v4-pro" PROMPT = "What is 17 + 26? Answer with just the number." @@ -67,7 +67,7 @@ class TestDeepSeekReasoningDisable: ) reasoning = _reasoning_content(response) assert reasoning, ( - "control case: deepseek-reasoner returned no reasoning_content with no " + "control case: the reasoning model returned no reasoning_content with no " f"disable param, so the disable assertions below can't be trusted: {response}" ) diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 157caedd561..128913802e2 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -72,7 +72,7 @@ class TestEmbeddingsEndpoint: model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="vertex_ai/gemini-embedding-2", + model="vertex_ai/text-embedding-005", vertex_project="os.environ/VERTEXAI_PROJECT", vertex_location="us-central1", ), diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py index 45861d1e93a..f7c23e46581 100644 --- a/tests/e2e/llm_translation/test_image_generation_e2e.py +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -54,7 +54,7 @@ class TestImageGeneration: model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="bedrock/amazon.titan-image-generator-v2:0", + model="bedrock/amazon.nova-canvas-v1:0", aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION", diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py index c3614251e77..c9f58b2c03c 100644 --- a/tests/e2e/llm_translation/test_rerank_e2e.py +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -60,7 +60,7 @@ class TestRerank: model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="bedrock/amazon.rerank-v1:0", + model="bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0", aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION",