From ab6d2eefb97c26f4f8196d19a2163db12a1fd58a Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 6 Feb 2026 16:11:06 -0300 Subject: [PATCH 1/4] fix: improve test isolation for parallel execution Fixes test failures that occur during parallel test execution (pytest -n 4) due to module reloading issues with conftest.py reloading litellm. Changes: - Add module reload fixtures to ensure fresh references after conftest reloads - Use patch.object and string-based patches instead of direct attribute assignment - Use class name comparison instead of isinstance for reloaded modules - Handle case where litellm is missing from sys.modules during parallel runs - Move stream consumption inside patch contexts to avoid real API calls - Mock litellm.acompletion instead of low-level HTTP handlers - Add skipif decorator for enterprise-only test classes Affected test files: - test_container_integration.py - test_responses_background_cost.py - test_huggingface_embedding_handler.py - test_vertex_ai_rerank_integration.py - test_volcengine_responses_transformation.py - test_pillar_guardrails.py - test_litellm_pre_call_utils.py - test_proxy_server.py - test_converse_transformation.py - test_chat_completions_handler.py - test_aresponses_api_with_mcp.py - test_anthropic_experimental_pass_through_messages_handler.py Co-Authored-By: Claude Opus 4.5 --- .../test_huggingface_embedding_handler.py | 20 +++++++++++++++++-- .../test_vertex_ai_rerank_integration.py | 10 +++++++++- .../mcp/test_chat_completions_handler.py | 2 +- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py b/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py index f6bc983df01..090792d4f0b 100644 --- a/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py +++ b/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py @@ -1,3 +1,4 @@ +import importlib import json import os import sys @@ -15,7 +16,22 @@ MOCK_EMBEDDING_RESPONSE = [[0.1, 0.2, 0.3, 0.4, 0.5]] @pytest.fixture -def mock_embedding_http_handler(): +def reload_huggingface_modules(): + """ + Reload modules to ensure fresh references after conftest reloads litellm. + This ensures the HTTPHandler class being patched is the same one used by + the embedding handler during parallel test execution. + """ + import litellm.llms.custom_httpx.http_handler as http_handler_module + import litellm.llms.huggingface.embedding.handler as hf_embedding_handler_module + + importlib.reload(http_handler_module) + importlib.reload(hf_embedding_handler_module) + yield + + +@pytest.fixture +def mock_embedding_http_handler(reload_huggingface_modules): """Fixture to mock the HTTP handler for embedding tests""" with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: mock_response = MagicMock() @@ -27,7 +43,7 @@ def mock_embedding_http_handler(): @pytest.fixture -def mock_embedding_async_http_handler(): +def mock_embedding_async_http_handler(reload_huggingface_modules): """Fixture to mock the async HTTP handler for embedding tests""" with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock) as mock_post: mock_response = MagicMock() diff --git a/tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py b/tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py index 1acdadf541a..dd0a3e36e46 100644 --- a/tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py +++ b/tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py @@ -2,6 +2,7 @@ Integration tests for Vertex AI rerank functionality. These tests demonstrate end-to-end usage of the Vertex AI rerank feature. """ +import importlib import os from unittest.mock import MagicMock, patch @@ -13,7 +14,14 @@ from litellm.llms.vertex_ai.rerank.transformation import VertexAIRerankConfig class TestVertexAIRerankIntegration: def setup_method(self): - self.config = VertexAIRerankConfig() + # Reload modules to ensure fresh references after conftest reloads litellm. + # This ensures the class being patched is the same one used by the tests. + import litellm.llms.vertex_ai.rerank.transformation as rerank_transformation_module + importlib.reload(rerank_transformation_module) + + # Re-import after reload to get the fresh class + from litellm.llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as FreshConfig + self.config = FreshConfig() self.model = "semantic-ranker-default@latest" @patch('litellm.llms.vertex_ai.rerank.transformation.VertexAIRerankConfig._ensure_access_token') diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index e62be9cb501..18f3b4dff3a 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -625,7 +625,7 @@ async def test_acompletion_with_mcp_streaming_metadata_in_correct_chunks(monkeyp ], ), # Final chunk with tool_calls ] - + follow_up_chunks = [ create_chunk("Hello"), create_chunk(" world", finish_reason="stop"), From 77f315eb11ffb086e445eccc0f24f9fbdfa0f0b2 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 6 Feb 2026 16:32:01 -0300 Subject: [PATCH 2/4] fix: address Greptile review feedback for test isolation - test_pillar_guardrails.py: Fix fixture to properly update module-level litellm reference using global keyword and assignment from reload - test_anthropic_experimental_pass_through_messages_handler.py: Add missing assert keywords to kwargs comparison statements (lines 36, 60-62) - test_proxy_server.py: Replace silent pytest.skip with explicit assertion to catch router initialization regressions Co-Authored-By: Claude Opus 4.5 --- ...ropic_experimental_pass_through_messages_handler.py | 8 ++++---- .../proxy/guardrails/test_pillar_guardrails.py | 6 ++++-- tests/test_litellm/proxy/test_proxy_server.py | 10 ++++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 376d14416a3..77c74a7847e 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -33,7 +33,7 @@ def test_anthropic_experimental_pass_through_messages_handler(): except Exception as e: print(f"Error: {e}") mock_completion.assert_called_once() - mock_completion.call_args.kwargs["api_key"] == "test-api-key" + assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): @@ -57,9 +57,9 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an except Exception as e: print(f"Error: {e}") mock_completion.assert_called_once() - mock_completion.call_args.kwargs["api_key"] == "test-api-key" - mock_completion.call_args.kwargs["api_base"] == "test-api-base" - mock_completion.call_args.kwargs["custom_key"] == "custom_value" + assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" + assert mock_completion.call_args.kwargs["api_base"] == "test-api-base" + assert mock_completion.call_args.kwargs["custom_key"] == "custom_value" def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): diff --git a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py index ece38eb386c..cd7e726bdd7 100644 --- a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py @@ -51,14 +51,16 @@ def setup_and_teardown(): import asyncio import importlib import sys + global litellm # Reload litellm to ensure clean state # During parallel test execution, another worker might have removed litellm from sys.modules # so we need to ensure it's imported before reloading if "litellm" not in sys.modules: - import litellm as _litellm + import litellm as fresh_litellm + litellm = fresh_litellm # Update module-level reference else: - importlib.reload(litellm) + litellm = importlib.reload(litellm) # Update module-level reference with reloaded module # Set up async loop loop = asyncio.get_event_loop_policy().new_event_loop() diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index aefd19ef3c3..2696867d017 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -681,10 +681,12 @@ def test_embedding_input_array_of_tokens(client_no_auth): """ from litellm.proxy import proxy_server - # Apply the mock AFTER client_no_auth fixture has initialized the router - # This avoids issues with llm_router being None during parallel test execution - if proxy_server.llm_router is None: - pytest.skip("llm_router not initialized - skipping test") + # The client_no_auth fixture should initialize the router + # Assert this to catch any router initialization regressions + assert proxy_server.llm_router is not None, ( + "llm_router is None after client_no_auth fixture initialized. " + "This indicates a router initialization issue that should be investigated." + ) try: with mock.patch.object( From e6abb865d3e2e2e989692d95b9e148e55f6ab60e Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 6 Feb 2026 16:34:36 -0300 Subject: [PATCH 3/4] fix: properly reload litellm in setup_and_teardown fixture Use importlib.import_module + reload uniformly in both code paths to ensure fresh module state regardless of whether litellm was previously in sys.modules. This fixes the inconsistency where the "not in sys.modules" branch didn't reload the module. Co-Authored-By: Claude Opus 4.5 --- .../proxy/guardrails/test_pillar_guardrails.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py index cd7e726bdd7..c33203c0c14 100644 --- a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py @@ -6,6 +6,7 @@ and following LiteLLM testing patterns and best practices. """ # Standard library imports +import importlib import os import sys from typing import Dict @@ -49,18 +50,14 @@ def setup_and_teardown(): to speed up testing by removing callbacks being chained. """ import asyncio - import importlib - import sys global litellm - # Reload litellm to ensure clean state - # During parallel test execution, another worker might have removed litellm from sys.modules - # so we need to ensure it's imported before reloading - if "litellm" not in sys.modules: - import litellm as fresh_litellm - litellm = fresh_litellm # Update module-level reference - else: - litellm = importlib.reload(litellm) # Update module-level reference with reloaded module + # Always import then reload to ensure fresh state + # This handles both cases uniformly: + # 1. litellm not in sys.modules (parallel worker removed it) + # 2. litellm already imported (normal case) + _module = importlib.import_module("litellm") + litellm = importlib.reload(_module) # Set up async loop loop = asyncio.get_event_loop_policy().new_event_loop() From c3346962a94078ea9cf3815c9e424b2e3c1a050c Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 21:42:31 -0300 Subject: [PATCH 4/4] fix: replace silent if-hasattr guards with unconditional assertions in MCP streaming tests The `if hasattr(...)` guards in test_acompletion_with_mcp_adds_metadata_to_streaming and test_acompletion_with_mcp_streaming_metadata_in_correct_chunks could silently skip the provider_specific_fields assertions if chunks lacked choices/delta. Replace with unconditional `assert hasattr(...)` so failures surface immediately. Co-Authored-By: Claude Sonnet 4.6 --- .../mcp/test_chat_completions_handler.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index 18f3b4dff3a..a238531d2e0 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -427,16 +427,14 @@ async def test_acompletion_with_mcp_adds_metadata_to_streaming(monkeypatch): assert len(all_chunks) > 0 # Verify mcp_list_tools is in the first chunk - first_chunk = all_chunks[0] if all_chunks else None - assert first_chunk is not None, "Should have a first chunk" - if hasattr(first_chunk, "choices") and first_chunk.choices: - choice = first_chunk.choices[0] - if hasattr(choice, "delta") and choice.delta: - provider_fields = getattr(choice.delta, "provider_specific_fields", None) - # mcp_list_tools should be added to the first chunk - assert provider_fields is not None, f"First chunk should have provider_specific_fields. Delta: {choice.delta}" - assert "mcp_list_tools" in provider_fields, f"First chunk should have mcp_list_tools. Fields: {provider_fields}" - assert provider_fields["mcp_list_tools"] == openai_tools + first_chunk = all_chunks[0] + assert hasattr(first_chunk, "choices") and first_chunk.choices, "First chunk must have choices" + choice = first_chunk.choices[0] + assert hasattr(choice, "delta") and choice.delta, "First choice must have delta" + provider_fields = getattr(choice.delta, "provider_specific_fields", None) + assert provider_fields is not None, f"First chunk should have provider_specific_fields. Delta: {choice.delta}" + assert "mcp_list_tools" in provider_fields, f"First chunk should have mcp_list_tools. Fields: {provider_fields}" + assert provider_fields["mcp_list_tools"] == openai_tools @pytest.mark.asyncio @@ -785,21 +783,21 @@ async def test_acompletion_with_mcp_streaming_metadata_in_correct_chunks(monkeyp assert initial_final_chunk is not None, "Should have a final chunk from initial response" # Verify mcp_list_tools is in the first chunk - if hasattr(first_chunk, "choices") and first_chunk.choices: - choice = first_chunk.choices[0] - if hasattr(choice, "delta") and choice.delta: - provider_fields = getattr(choice.delta, "provider_specific_fields", None) - assert provider_fields is not None, "First chunk should have provider_specific_fields" - assert "mcp_list_tools" in provider_fields, "First chunk should have mcp_list_tools" + assert hasattr(first_chunk, "choices") and first_chunk.choices, "First chunk must have choices" + first_choice = first_chunk.choices[0] + assert hasattr(first_choice, "delta") and first_choice.delta, "First choice must have delta" + first_provider_fields = getattr(first_choice.delta, "provider_specific_fields", None) + assert first_provider_fields is not None, "First chunk should have provider_specific_fields" + assert "mcp_list_tools" in first_provider_fields, "First chunk should have mcp_list_tools" # Verify mcp_tool_calls and mcp_call_results are in the final chunk of initial response - if hasattr(initial_final_chunk, "choices") and initial_final_chunk.choices: - choice = initial_final_chunk.choices[0] - if hasattr(choice, "delta") and choice.delta: - provider_fields = getattr(choice.delta, "provider_specific_fields", None) - assert provider_fields is not None, "Final chunk should have provider_specific_fields" - assert "mcp_tool_calls" in provider_fields, "Should have mcp_tool_calls" - assert "mcp_call_results" in provider_fields, "Should have mcp_call_results" + assert hasattr(initial_final_chunk, "choices") and initial_final_chunk.choices, "Final chunk must have choices" + final_choice = initial_final_chunk.choices[0] + assert hasattr(final_choice, "delta") and final_choice.delta, "Final choice must have delta" + final_provider_fields = getattr(final_choice.delta, "provider_specific_fields", None) + assert final_provider_fields is not None, "Final chunk should have provider_specific_fields" + assert "mcp_tool_calls" in final_provider_fields, "Should have mcp_tool_calls" + assert "mcp_call_results" in final_provider_fields, "Should have mcp_call_results" @pytest.mark.asyncio