diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index ef8c7170d0f..9a79db89164 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -30,15 +30,6 @@ async def test_azure_health_check(): # asyncio.run(test_azure_health_check()) -@pytest.mark.asyncio -async def test_text_completion_health_check(): - response = await litellm.ahealth_check( - model_params={"model": "gpt-3.5-turbo-instruct"}, - mode="completion", - prompt="What's the weather in SF?", - ) - print(f"response: {response}") - return response @pytest.mark.asyncio @@ -333,17 +324,3 @@ async def test_timeout_does_not_cancel_other_health_checks(): assert "openai/fast-model" in healthy_models assert "openai/slow-model" in unhealthy_models - - -@pytest.mark.asyncio -async def test_ahealth_check_ocr(): - litellm.turn_on_debug() - response = await litellm.ahealth_check( - model_params={ - "model": "mistral/mistral-ocr-latest", - "api_key": os.getenv("MISTRAL_API_KEY"), - }, - mode="ocr", - ) - print(response) - return response diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index aadb093c050..4e2eecd2f31 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -29,7 +29,6 @@ from openai import OpenAI sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from tests._live_test_helpers import _skip_live_prompt_caching_test # noqa: E402 def _usage_format_tests(usage: litellm.Usage): @@ -640,15 +639,6 @@ class BaseLLMChatTest(ABC): except litellm.InternalServerError: pytest.skip("Model is overloaded") - @pytest.mark.flaky(retries=6, delay=1) - def test_json_response_pydantic_obj_nested_obj(self): - litellm.set_verbose = True - from pydantic import BaseModel - from litellm.utils import supports_response_schema - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - @pytest.mark.flaky(retries=6, delay=1) def test_json_response_nested_pydantic_obj(self): from pydantic import BaseModel @@ -845,11 +835,6 @@ class BaseLLMChatTest(ABC): ], } - @abstractmethod - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - @pytest.mark.parametrize("detail", [None, "low", "high"]) @pytest.mark.parametrize( "image_url", @@ -962,108 +947,6 @@ class BaseLLMChatTest(ABC): assert response is not None - @pytest.mark.flaky(retries=4, delay=1) - def test_prompt_caching(self): - _skip_live_prompt_caching_test() - print("test_prompt_caching") - litellm.set_verbose = True - from litellm.utils import supports_prompt_caching - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - base_completion_call_args = self.get_base_completion_call_args() - if not supports_prompt_caching(base_completion_call_args["model"], None): - print("Model does not support prompt caching") - pytest.skip("Model does not support prompt caching") - - uuid_str = str(uuid.uuid4()) - messages = [ - # System Message - { - "role": "system", - "content": [ - { - "type": "text", - "text": "Here is the full text of a complex legal agreement {}".format( - uuid_str - ) - * 400, - "cache_control": {"type": "ephemeral"}, - } - ], - }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - { - "role": "assistant", - "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", - }, - # The final turn is marked with cache-control, for continuing in followups. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - ] - - try: - ## call 1 - response = self.completion_function( - **base_completion_call_args, - messages=messages, - max_tokens=10, - ) - - print("response=", response) - - initial_cost = response._hidden_params["response_cost"] - ## call 2 - response = self.completion_function( - **base_completion_call_args, - messages=messages, - max_tokens=10, - ) - - time.sleep(1) - - cached_cost = response._hidden_params["response_cost"] - - assert ( - cached_cost <= initial_cost - ), "Cached cost={} should be less than initial cost={}".format( - cached_cost, initial_cost - ) - - _usage_format_tests(response.usage) - - print("response=", response) - print("response.usage=", response.usage) - - _usage_format_tests(response.usage) - - assert "prompt_tokens_details" in response.usage - if response.usage.prompt_tokens_details is not None: - assert ( - response.usage.prompt_tokens_details.cached_tokens > 0 - ), f"cached_tokens={response.usage.prompt_tokens_details.cached_tokens} should be greater than 0. Got usage={response.usage}" - except litellm.InternalServerError as e: - print("InternalServerError", e) - @pytest.fixture def pdf_messages(self): import base64 diff --git a/tests/llm_translation/realtime/base_realtime_tests.py b/tests/llm_translation/realtime/base_realtime_tests.py index fd04dcb035f..82665235f67 100644 --- a/tests/llm_translation/realtime/base_realtime_tests.py +++ b/tests/llm_translation/realtime/base_realtime_tests.py @@ -439,17 +439,3 @@ class BaseRealtimeTest(ABC): assert websocket_client.connection_successful, "Failed to establish connection" assert websocket_client.sent_user_message, "Failed to send user message" - - def test_query_params_construction(self): - """Test that query params are constructed correctly""" - from litellm.types.realtime import RealtimeQueryParams - - # Strip provider prefix from model name - model_name = self.get_model() - if "/" in model_name: - model_name = model_name.split("/", 1)[1] - - query_params: RealtimeQueryParams = {"model": model_name} - - assert "model" in query_params - assert query_params["model"] == model_name diff --git a/tests/llm_translation/realtime/test_openai_realtime.py b/tests/llm_translation/realtime/test_openai_realtime.py index 30b47b8d3ea..f137e9df49b 100644 --- a/tests/llm_translation/realtime/test_openai_realtime.py +++ b/tests/llm_translation/realtime/test_openai_realtime.py @@ -136,33 +136,3 @@ async def test_openai_realtime_direct_call_no_intent(): ), "session.created response missing session object" assert "id" in session_message["session"], "Session object missing id field" assert "model" in session_message["session"], "Session object missing model field" - - -def test_realtime_query_params_construction(): - """ - Test that query params are constructed correctly by the proxy server logic - """ - from litellm.types.realtime import RealtimeQueryParams - - # Test case 1: intent is None (should not be included) - model = "gpt-4o-realtime-preview" - intent = None - - query_params: RealtimeQueryParams = {"model": model} - if intent is not None: - query_params["intent"] = intent - - assert "model" in query_params - assert query_params["model"] == model - assert "intent" not in query_params - - # Test case 2: intent is provided (should be included) - intent = "chat" - query_params2: RealtimeQueryParams = {"model": model} - if intent is not None: - query_params2["intent"] = intent - - assert "model" in query_params2 - assert query_params2["model"] == model - assert "intent" in query_params2 - assert query_params2["intent"] == intent diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 98f70c2d9d7..5f17b44fd3d 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -315,15 +315,6 @@ class TestAnthropicCompletion(BaseLLMChatTest, BaseAnthropicChatTest): "thinking": {"type": "enabled", "budget_tokens": 16000}, } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_anthropic_tool_invoke, - ) - - result = convert_to_anthropic_tool_invoke([tool_call_no_arguments]) - print(result) - def test_tool_call_and_json_response_format(self): """ Test that the tool call and JSON response format is supported by the LLM API diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index 6a455beb09b..4d5ac43ab6b 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -31,16 +31,9 @@ class TestAzureOpenAIO3Mini(BaseOSeriesModelsTest, BaseLLMChatTest): api_version="2024-02-15-preview", ) - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - def test_basic_tool_calling(self): pass - def test_prompt_caching(self): - """Temporary override. o1 prompt caching is not working.""" - pass class TestAzureOpenAIO3(BaseOSeriesModelsTest): diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index cb292ffdab1..6d8e8a431db 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -709,16 +709,6 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - - def test_prompt_caching(self): - """ - Remove override once we have access to Bedrock prompt caching - """ - pass - def test_completion_cost(self): """ Test if region models info is correctly used for cost calculation. Using the base model info for cost calculation. @@ -770,9 +760,6 @@ class TestBedrockConverseChatNormal(BaseLLMChatTest): "aws_region_name": "us-east-1", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass class TestBedrockConverseNovaTestSuite(BaseLLMChatTest): @@ -789,14 +776,7 @@ class TestBedrockConverseNovaTestSuite(BaseLLMChatTest): "aws_region_name": "us-east-1", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - def test_prompt_caching(self): - """ - TODO: Ensure this test passes our base llm test suite - """ class TestBedrockRerank(BaseLLMRerankTest): diff --git a/tests/llm_translation/test_bedrock_gpt_oss.py b/tests/llm_translation/test_bedrock_gpt_oss.py index a0b26d4b674..e0d7b1b904f 100644 --- a/tests/llm_translation/test_bedrock_gpt_oss.py +++ b/tests/llm_translation/test_bedrock_gpt_oss.py @@ -9,10 +9,6 @@ class TestBedrockGPTOSS(BaseLLMChatTest): "model": "bedrock/converse/openai.gpt-oss-20b-1:0", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - def test_function_calling_with_tool_response(self): """Bedrock GPT-OSS intermittently emits truncated toolUse.input deltas on the live endpoint, which makes the inherited live integration test flaky. @@ -23,13 +19,6 @@ class TestBedrockGPTOSS(BaseLLMChatTest): """ pass - - def test_prompt_caching(self): - """ - Remove override once we have access to Bedrock prompt caching - """ - pass - async def test_completion_cost(self): """ Bedrock GPT-OSS models are flaky and occasionally report 0 token counts in api response diff --git a/tests/llm_translation/test_bedrock_invoke_tests.py b/tests/llm_translation/test_bedrock_invoke_tests.py index fd92586cb7b..584b0ef341f 100644 --- a/tests/llm_translation/test_bedrock_invoke_tests.py +++ b/tests/llm_translation/test_bedrock_invoke_tests.py @@ -24,10 +24,6 @@ class TestBedrockInvokeClaudeJson(BaseLLMChatTest): "model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - @pytest.mark.parametrize( "image_url, detail", [ @@ -54,10 +50,6 @@ class TestBedrockInvokeNovaJson(BaseLLMChatTest): "model": "bedrock/invoke/us.amazon.nova-micro-v1:0", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - @pytest.fixture(autouse=True) def skip_non_json_tests(self, request): if not "json" in request.function.__name__.lower(): diff --git a/tests/llm_translation/test_bedrock_llama.py b/tests/llm_translation/test_bedrock_llama.py index 9fbf108fbc4..9c83c28bbe0 100644 --- a/tests/llm_translation/test_bedrock_llama.py +++ b/tests/llm_translation/test_bedrock_llama.py @@ -9,9 +9,6 @@ class TestBedrockTestSuite(BaseLLMChatTest): test_empty_tools = None test_function_calling_with_tool_response = None - def test_tool_call_no_arguments(self, tool_call_no_arguments): - pass - def get_base_completion_call_args(self) -> dict: litellm.turn_on_debug() return { diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index 61de9e456a8..ac1a8363845 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -38,22 +38,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): "model": "bedrock/invoke/moonshot.kimi-k2-thinking", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly.""" - pass - class TestBedrockMoonshotToolCalling: """Unit tests for tool calling functionality.""" - - def test_tool_response_message_format(self): - """Test that tool response messages are formatted correctly.""" - tool_response_message = { - "role": "tool", - "tool_call_id": "call_123", - "content": json.dumps({"temperature": 72, "condition": "sunny"}), - } - - assert tool_response_message["role"] == "tool" - assert "tool_call_id" in tool_response_message - assert "content" in tool_response_message diff --git a/tests/llm_translation/test_bedrock_nova_json.py b/tests/llm_translation/test_bedrock_nova_json.py index be8fd321316..8adfef50618 100644 --- a/tests/llm_translation/test_bedrock_nova_json.py +++ b/tests/llm_translation/test_bedrock_nova_json.py @@ -25,15 +25,7 @@ class TestBedrockNovaJson(BaseLLMChatTest): def test_json_response_nested_json_schema(self): pass - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - def test_prompt_caching(self): - """ - Remove override once we have access to Bedrock prompt caching - """ - pass # @pytest.fixture(autouse=True) # def skip_non_json_tests(self, request): diff --git a/tests/llm_translation/test_evals_api.py b/tests/llm_translation/test_evals_api.py index ba6b5edf3cd..0dd3b88382e 100644 --- a/tests/llm_translation/test_evals_api.py +++ b/tests/llm_translation/test_evals_api.py @@ -260,23 +260,6 @@ class BaseEvalsAPITest(ABC): assert response.name == updated_name print(f"Updated eval: {response}") - def test_delete_eval(self): - """ - Test deleting an evaluation. - - Real delete coverage now lives in the ``managed_eval`` fixture - teardown and in ``test_create_eval``'s ``finally`` block, so - this stays a no-op skip rather than creating a fresh resource - just to delete it. - """ - custom_llm_provider = self.get_custom_llm_provider() - api_key = self.get_api_key() - api_base = self.get_api_base() - - if not api_key: - pytest.skip(f"No API key provided for {custom_llm_provider}") - - pytest.skip("Delete is exercised via managed_eval fixture teardown.") class TestOpenAIEvalsAPI(BaseEvalsAPITest): diff --git a/tests/llm_translation/test_groq.py b/tests/llm_translation/test_groq.py index 01be9c745f8..55b1b9bc7eb 100644 --- a/tests/llm_translation/test_groq.py +++ b/tests/llm_translation/test_groq.py @@ -19,9 +19,5 @@ class TestGroq(BaseLLMChatTest): "model": "groq/openai/gpt-oss-120b", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - def test_tool_call_with_empty_enum_property(self): pass diff --git a/tests/llm_translation/test_mistral_api.py b/tests/llm_translation/test_mistral_api.py index e0490882ea3..3e709840daa 100644 --- a/tests/llm_translation/test_mistral_api.py +++ b/tests/llm_translation/test_mistral_api.py @@ -29,7 +29,3 @@ class TestMistralCompletion(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: litellm.set_verbose = True return {"model": "mistral/mistral-medium-latest"} - - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index e4410e13e6f..b4e8b87c90b 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -215,16 +215,6 @@ class TestOpenAIChatCompletion(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: return {"model": "gpt-4o-mini"} - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - - def test_prompt_caching(self): - """ - Works locally but CI/CD is failing this test. Temporary skip to push out a new release. - """ - pass - @pytest.mark.parametrize("model", ["o1", "o3-mini"]) def test_o1_parallel_tool_calls(model): diff --git a/tests/llm_translation/test_openai_o1.py b/tests/llm_translation/test_openai_o1.py index 30e835c6a19..61b17dfecf1 100644 --- a/tests/llm_translation/test_openai_o1.py +++ b/tests/llm_translation/test_openai_o1.py @@ -24,13 +24,7 @@ class TestOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest): return OpenAI(api_key="fake-api-key") - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - def test_prompt_caching(self): - """Temporary override. o1 prompt caching is not working.""" - pass class TestOpenAIO3(BaseOSeriesModelsTest, BaseLLMChatTest): @@ -47,13 +41,7 @@ class TestOpenAIO3(BaseOSeriesModelsTest, BaseLLMChatTest): return OpenAI(api_key="fake-api-key") - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - def test_prompt_caching(self): - """Override, as o3 prompt caching is flaky""" - pass def test_o3_reasoning_effort(): diff --git a/tests/llm_translation/test_router_llm_translation_tests.py b/tests/llm_translation/test_router_llm_translation_tests.py index 10807adf356..a5c63530e06 100644 --- a/tests/llm_translation/test_router_llm_translation_tests.py +++ b/tests/llm_translation/test_router_llm_translation_tests.py @@ -41,16 +41,6 @@ class TestRouterLLMTranslation(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: return {"model": "gpt-4o-mini"} - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - - def test_prompt_caching(self): - """ - Works locally but CI/CD is failing this test. Temporary skip to push out a new release. - """ - pass - def test_router_azure_acompletion(): # [PROD TEST CASE] diff --git a/tests/llm_translation/test_together_ai.py b/tests/llm_translation/test_together_ai.py index 885f0ea5917..a203c9edcfe 100644 --- a/tests/llm_translation/test_together_ai.py +++ b/tests/llm_translation/test_together_ai.py @@ -27,7 +27,3 @@ class TestTogetherAI(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: litellm.set_verbose = True return {"model": cheapest_together_chat_model(function_calling=True, response_schema=True)} - - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass diff --git a/tests/llm_translation/test_xai.py b/tests/llm_translation/test_xai.py index 7a9e4debbcd..37c96531217 100644 --- a/tests/llm_translation/test_xai.py +++ b/tests/llm_translation/test_xai.py @@ -53,10 +53,6 @@ class TestXAIChat(BaseLLMChatTest): "model": "xai/grok-3-mini-beta", } - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - test_web_search = None diff --git a/tests/local_testing/test_lunary.py b/tests/local_testing/test_lunary.py index f561ce00f3e..c2f67a137c0 100644 --- a/tests/local_testing/test_lunary.py +++ b/tests/local_testing/test_lunary.py @@ -9,47 +9,10 @@ litellm.success_callback = ["lunary"] litellm.set_verbose = True -def test_lunary_logging(): - try: - response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "what llm are u"}], - max_tokens=10, - temperature=0.2, - user="test-user", - ) - print(response) - except Exception as e: - print(e) -def test_lunary_template(): - import lunary - - try: - template = lunary.render_template("test-template", {"question": "Hello!"}) - response = completion(**template) - print(response) - except Exception as e: - print(e) -def test_lunary_logging_with_metadata(): - try: - response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "what llm are u"}], - max_tokens=10, - temperature=0.2, - metadata={ - "run_name": "litellmRUN", - "project_name": "litellm-completion", - "tags": ["tag1", "tag2"], - }, - ) - print(response) - except Exception as e: - print(e) def test_lunary_with_tools(): @@ -93,22 +56,3 @@ def test_lunary_with_tools(): assert response.choices[0].message.tool_calls assert all(call.function.name == "get_current_weather" for call in response.choices[0].message.tool_calls) print("\nLLM Response:\n", response.choices[0].message) - - -def test_lunary_logging_with_streaming_and_metadata(): - try: - response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "what llm are u"}], - max_tokens=10, - temperature=0.2, - metadata={ - "run_name": "litellmRUN", - "project_name": "litellm-completion", - }, - stream=True, - ) - for chunk in response: - continue - except Exception as e: - print(e) diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index 2d317d8e707..a656b1e991f 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -450,50 +450,6 @@ def test_function_calling(): # test_acompletion_on_router() -def test_function_calling_on_router(): - try: - litellm.set_verbose = True - model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - ] - function1 = [ - { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - } - ] - router = Router( - model_list=model_list, - redis_host=os.getenv("REDIS_HOST"), - redis_password=os.getenv("REDIS_PASSWORD"), - redis_port=os.getenv("REDIS_PORT"), - ) - messages = [{"role": "user", "content": "what's the weather in boston"}] - response = router.completion( - model="gpt-3.5-turbo", messages=messages, functions=function1 - ) - print(f"final returned response: {response}") - router.reset() - assert isinstance(response["choices"][0]["message"]["function_call"], dict) - except Exception as e: - print(f"An exception occurred: {e}") # test_function_calling_on_router() diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index ce85c0447b3..c04221167f2 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -54,85 +54,6 @@ kwargs = { "messages": [{"role": "user", "content": "Hey, how's it going?"}], } -def test_sync_fallbacks(): - try: - model_list = [ - { # list of model deployments - "model_name": "azure/gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/gpt-4.1-mini", - "api_key": "bad-key", - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { # list of model deployments - "model_name": "azure/gpt-3.5-turbo-context-fallback", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_AI_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "azure/gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": "bad-key", - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - "tpm": 1000000, - "rpm": 9000, - }, - { - "model_name": "gpt-3.5-turbo-16k", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo-16k", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - "tpm": 1000000, - "rpm": 9000, - }, - ] - - litellm.set_verbose = True - customHandler = MyCustomHandler() - litellm.callbacks = [customHandler] - router = Router( - model_list=model_list, - fallbacks=[{"azure/gpt-3.5-turbo": ["gpt-3.5-turbo"]}], - context_window_fallbacks=[ - {"azure/gpt-3.5-turbo-context-fallback": ["gpt-3.5-turbo-16k"]}, - {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}, - ], - set_verbose=False, - ) - response = router.completion(**kwargs) - print(f"response: {response}") - time.sleep(0.05) # allow a delay as success_callbacks are on a separate thread - assert ( - customHandler.previous_models == 3 - ) # 1 init call + 2 retries (fallback not counted as previous) - - print("Passed ! Test router_fallbacks: test_sync_fallbacks()") - router.reset() - except Exception as e: - print(e) # test_sync_fallbacks() @@ -489,83 +410,6 @@ async def test_dynamic_fallbacks_async(): # asyncio.run(test_dynamic_fallbacks_async()) -def test_sync_fallbacks_streaming(): - try: - model_list = [ - { # list of model deployments - "model_name": "azure/gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/gpt-4.1-mini", - "api_key": "bad-key", - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { # list of model deployments - "model_name": "azure/gpt-3.5-turbo-context-fallback", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_AI_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "azure/gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": "bad-key", - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - "tpm": 1000000, - "rpm": 9000, - }, - { - "model_name": "gpt-3.5-turbo-16k", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo-16k", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - "tpm": 1000000, - "rpm": 9000, - }, - ] - - litellm.set_verbose = True - customHandler = MyCustomHandler() - litellm.callbacks = [customHandler] - router = Router( - model_list=model_list, - fallbacks=[{"azure/gpt-3.5-turbo": ["gpt-3.5-turbo"]}], - context_window_fallbacks=[ - {"azure/gpt-3.5-turbo-context-fallback": ["gpt-3.5-turbo-16k"]}, - {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}, - ], - set_verbose=False, - ) - response = router.completion(**kwargs, stream=True) - print(f"response: {response}") - time.sleep(0.05) # allow a delay as success_callbacks are on a separate thread - assert customHandler.previous_models == 1 # 0 retries, 1 fallback - - print("Passed ! Test router_fallbacks: test_sync_fallbacks()") - router.reset() - except Exception as e: - print(e) @pytest.mark.asyncio async def test_async_fallbacks_max_retries_per_request(): @@ -777,94 +621,6 @@ def test_ausage_based_routing_fallbacks(): except Exception as e: pytest.fail(f"An exception occurred {e}") -def test_custom_cooldown_times(): - try: - # set, custom_cooldown. Failed model in cooldown_models, after custom_cooldown, the failed model is no longer in cooldown_models - - model_list = [ - { # list of model deployments - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/gpt-4.1-mini", - "api_key": "bad-key", - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 24000000, - }, - { # list of model deployments - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_AI_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - }, - "tpm": 1, - }, - ] - - litellm.set_verbose = False - - router = Router( - model_list=model_list, - set_verbose=True, - debug_level="INFO", - cooldown_time=0.1, - redis_host=os.getenv("REDIS_HOST"), - redis_password=os.getenv("REDIS_PASSWORD"), - redis_port=int(os.getenv("REDIS_PORT")), - ) - - # make a request - expect it to fail - try: - response = router.completion( - model="gpt-3.5-turbo", - messages=[ - { - "content": "Tell me a joke.", - "role": "user", - } - ], - ) - except Exception: - pass - - # expect 1 model to be in cooldown models - cooldown_deployments = router._get_cooldown_deployments() - print("cooldown_deployments after failed call: ", cooldown_deployments) - assert ( - len(cooldown_deployments) == 1 - ), "Expected 1 model to be in cooldown models" - - selected_cooldown_model = cooldown_deployments[0] - - # wait for 1/2 of cooldown time - time.sleep(router.cooldown_time / 2) - - # expect cooldown model to still be in cooldown models - cooldown_deployments = router._get_cooldown_deployments() - print( - "cooldown_deployments after waiting 1/2 of cooldown: ", cooldown_deployments - ) - assert ( - len(cooldown_deployments) == 1 - ), "Expected 1 model to be in cooldown models" - - # wait for 1/2 of cooldown time again, now we've waited for full cooldown - time.sleep(router.cooldown_time / 2) - - # expect cooldown model to be removed from cooldown models - cooldown_deployments = router._get_cooldown_deployments() - print( - "cooldown_deployments after waiting cooldown time: ", cooldown_deployments - ) - assert ( - len(cooldown_deployments) == 0 - ), "Expected 0 models to be in cooldown models" - - except Exception as e: - print(e) @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 6dafae39d04..76301f69c3f 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -818,20 +818,6 @@ def test_openai_chat_completion_call(): print(f"complete response: {complete_response}") -def test_openai_chat_completion_complete_response_call(): - try: - complete_response = completion( - model="gpt-3.5-turbo", - messages=messages, - stream=True, - complete_response=True, - ) - print(f"complete response: {complete_response}") - except Exception: - print(f"error occurred: {traceback.format_exc()}") - pass - - @pytest.mark.parametrize( "model", [ @@ -925,90 +911,14 @@ def test_openai_stream_options_call_text_completion() -> None: assert any(chunk.choices[0].text for chunk in chunks) -def test_openai_text_completion_call(): - try: - litellm.set_verbose = True - response = completion( - model="gpt-3.5-turbo-instruct", messages=messages, stream=True - ) - complete_response = "" - start_time = time.time() - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - print(f"chunk: {chunk}") - complete_response += chunk - if finished: - break - # print(f'complete_chunk: {complete_response}') - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"complete response: {complete_response}") - except Exception: - print(f"error occurred: {traceback.format_exc()}") - pass - - -# # test on together ai completion call - starcoder -def test_together_ai_completion_call_mistral(): - try: - litellm.set_verbose = False - start_time = time.time() - response = completion( - model="together_ai/mistralai/Mistral-7B-Instruct-v0.2", - messages=messages, - logger_fn=logger_fn, - stream=True, - ) - complete_response = "" - print(f"returned response object: {response}") - has_finish_reason = False - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - has_finish_reason = finished - if finished: - break - complete_response += chunk - if has_finish_reason is False: - raise Exception("Finish reason not set for last chunk") - if complete_response == "": - raise Exception("Empty response received") - print(f"complete response: {complete_response}") - except Exception: - print(f"error occurred: {traceback.format_exc()}") - pass # # test on together ai completion call - starcoder -def test_together_ai_completion_call_starcoder_bad_key(): - try: - api_key = "bad-key" - start_time = time.time() - response = completion( - model="together_ai/bigcode/starcoder", - messages=messages, - stream=True, - api_key=api_key, - ) - complete_response = "" - has_finish_reason = False - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - has_finish_reason = finished - if finished: - break - complete_response += chunk - if has_finish_reason is False: - raise Exception("Finish reason not set for last chunk") - if complete_response == "": - raise Exception("Empty response received") - print(f"complete response: {complete_response}") - except BadRequestError as e: - pass - except Exception: - print(f"error occurred: {traceback.format_exc()}") - pass +# # test on together ai completion call - starcoder + + #### Test Function calling + streaming #### diff --git a/tests/local_testing/test_text_completion.py b/tests/local_testing/test_text_completion.py index 13bc2f652df..69064dfd1f4 100644 --- a/tests/local_testing/test_text_completion.py +++ b/tests/local_testing/test_text_completion.py @@ -2850,25 +2850,6 @@ def test_completion_text_003_prompt_array(): # asyncio.run(test_text_completion_async_stream()) -def test_async_text_completion(): - litellm.set_verbose = True - print("test_async_text_completion") - - async def test_get_response(): - try: - response = await litellm.atext_completion( - model="gpt-3.5-turbo-instruct", - prompt="good morning", - stream=False, - max_tokens=10, - ) - print(f"response: {response}") - except litellm.Timeout as e: - print(e) - except Exception as e: - print(e) - - asyncio.run(test_get_response()) # test_async_text_completion() diff --git a/tests/logging_callback_tests/base_test.py b/tests/logging_callback_tests/base_test.py index 68faf4bdb35..cc894498f5e 100644 --- a/tests/logging_callback_tests/base_test.py +++ b/tests/logging_callback_tests/base_test.py @@ -9,7 +9,6 @@ import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper -from litellm.types.utils import ModelResponse # test_example.py from abc import ABC, abstractmethod @@ -84,12 +83,3 @@ class BaseLoggingCallbackTest(ABC): ), service_tier=None, ) - - @abstractmethod - def test_parallel_tool_calls(self, mock_response_obj: ModelResponse): - """ - Check if parallel tool calls are correctly logged by Logging callback - - Relevant issue - https://github.com/BerriAI/litellm/issues/6677 - """ - pass diff --git a/tests/logging_callback_tests/test_datadog_llm_obs.py b/tests/logging_callback_tests/test_datadog_llm_obs.py deleted file mode 100644 index bed1a214b44..00000000000 --- a/tests/logging_callback_tests/test_datadog_llm_obs.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -Test the DataDogLLMObsLogger -""" - -import io - - - -import asyncio -import gzip -import json -import logging -import time -from unittest.mock import AsyncMock, patch - -import pytest - -import litellm -from litellm import completion -from litellm._logging import verbose_logger -from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger -from datetime import datetime, timedelta -from litellm.types.integrations.datadog_llm_obs import * -from litellm.types.utils import ( - StandardLoggingPayload, - StandardLoggingModelInformation, - StandardLoggingMetadata, - StandardLoggingHiddenParams, -) - -verbose_logger.setLevel(logging.DEBUG) - - -def create_standard_logging_payload() -> StandardLoggingPayload: - return StandardLoggingPayload( - id="test_id", - call_type="completion", - response_cost=0.1, - response_cost_failure_debug_info=None, - status="success", - total_tokens=30, - prompt_tokens=20, - completion_tokens=10, - startTime=1234567890.0, - endTime=1234567891.0, - completionStartTime=1234567890.5, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-5-mini", model_map_value=None - ), - model="gpt-5-mini", - model_id="model-123", - model_group="openai-gpt", - api_base="https://api.openai.com", - metadata=StandardLoggingMetadata( - user_api_key_hash="test_hash", - user_api_key_org_id=None, - user_api_key_alias="test_alias", - user_api_key_team_id="test_team", - user_api_key_user_id="test_user", - user_api_key_team_alias="test_team_alias", - spend_logs_metadata=None, - requester_ip_address="127.0.0.1", - requester_metadata=None, - ), - cache_hit=False, - cache_key=None, - saved_cache_cost=0.0, - request_tags=[], - end_user=None, - requester_ip_address="127.0.0.1", - messages=[{"role": "user", "content": "Hello, world!"}], - response={"choices": [{"message": {"content": "Hi there!"}}]}, - error_str=None, - model_parameters={"stream": True}, - hidden_params=StandardLoggingHiddenParams( - model_id="model-123", - cache_key=None, - api_base="https://api.openai.com", - response_cost="0.1", - additional_headers=None, - ), - ) - - -@pytest.mark.asyncio -async def test_datadog_llm_obs_logging(): - datadog_llm_obs_logger = DataDogLLMObsLogger() - litellm.callbacks = [datadog_llm_obs_logger] - litellm.set_verbose = True - - for _ in range(2): - response = await litellm.acompletion( - model="gpt-5.5", - messages=[{"role": "user", "content": "Hello testing dd llm obs!"}], - mock_response="hi", - ) - - print(response) - - await asyncio.sleep(6) diff --git a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py b/tests/openai_endpoints_tests/test_openai_batches_endpoint.py index ec265167271..bf36ce4d7d1 100644 --- a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py +++ b/tests/openai_endpoints_tests/test_openai_batches_endpoint.py @@ -9,7 +9,7 @@ from typing import Optional, List, Union from test_openai_files_endpoints import upload_file, delete_file import sys import time -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import patch BASE_URL = "http://localhost:4000" # Replace with your actual base URL @@ -139,81 +139,6 @@ def test_vertex_batches_endpoint(): pass -@pytest.mark.skip(reason="Local only test to verify if things work well") -@pytest.mark.asyncio -async def test_list_batches_with_target_model_names(): - """ - Unit test to verify that target_model_names query parameter is properly handled - in the list_batches endpoint - """ - - # Test data - target_model_names = "gpt-5.5,gpt-5-mini" - expected_model = "gpt-5.5" # Should use the first model from the comma-separated list - - # Mock response for list_batches - mock_batch_response = { - "object": "list", - "data": [ - { - "id": "batch_abc123", - "object": "batch", - "endpoint": "/v1/chat/completions", - "status": "validating", - "input_file_id": "file-abc123", - "completion_window": "24h", - "created_at": 1711471533, - "metadata": {}, - } - ], - "first_id": "batch_abc123", - "last_id": "batch_abc123", - "has_more": False, - } - - # Mock the request and FastAPI dependencies - mock_request = MagicMock() - mock_request.method = "GET" - mock_request.url.query = f"target_model_names={target_model_names}&limit=10" - - mock_fastapi_response = MagicMock() - mock_user_api_key_dict = MagicMock() - - # Mock _read_request_body to return our target_model_names - with ( - patch( - "litellm.proxy.batches_endpoints.endpoints._read_request_body" - ) as mock_read_body, - patch("litellm.proxy.proxy_server.llm_router") as mock_router, - ): - - mock_read_body.return_value = {"target_model_names": target_model_names} - mock_router.alist_batches = AsyncMock(return_value=mock_batch_response) - - # Import and call the function directly - from litellm.proxy.batches_endpoints.endpoints import list_batches - - response = await list_batches( - request=mock_request, - fastapi_response=mock_fastapi_response, - target_model_names=target_model_names, - limit=10, - user_api_key_dict=mock_user_api_key_dict, - ) - - # Verify that router.alist_batches was called with the correct model - mock_router.alist_batches.assert_called_once() - call_args = mock_router.alist_batches.call_args - - # Check that the model parameter was set to the first model in the list - assert call_args.kwargs["model"] == expected_model - assert call_args.kwargs["limit"] == 10 - - # Verify the response structure - assert response["object"] == "list" - assert len(response["data"]) > 0 - - @pytest.mark.asyncio async def test_batch_status_sync_from_provider_to_database(): """ diff --git a/tests/otel_tests/test_guardrails.py b/tests/otel_tests/test_guardrails.py index 3aa6b8cb2e7..36f2d019a34 100644 --- a/tests/otel_tests/test_guardrails.py +++ b/tests/otel_tests/test_guardrails.py @@ -72,62 +72,6 @@ async def generate_key( return await response.json() -@pytest.mark.asyncio -@pytest.mark.skip(reason="Aporia account disabled") -async def test_llm_guard_triggered_safe_request(): - """ - - Tests a request where no content mod is triggered - - Assert that the guardrails applied are returned in the response headers - """ - async with aiohttp.ClientSession() as session: - response, headers = await chat_completion( - session, - os.environ["LITELLM_MASTER_KEY"], - model="fake-openai-endpoint", - messages=[{"role": "user", "content": f"Hello what's the weather"}], - guardrails=[ - "aporia-post-guard", - "aporia-pre-guard", - ], - ) - await asyncio.sleep(3) - - print("response=", response, "response headers", headers) - - assert "x-litellm-applied-guardrails" in headers - - assert ( - headers["x-litellm-applied-guardrails"] - == "aporia-pre-guard,aporia-post-guard" - ) - - -@pytest.mark.asyncio -@pytest.mark.skip(reason="Aporia account disabled") -async def test_llm_guard_triggered(): - """ - - Tests a request where no content mod is triggered - - Assert that the guardrails applied are returned in the response headers - """ - async with aiohttp.ClientSession() as session: - with pytest.raises(Exception, match="Aporia detected and blocked PII") as exc_info: - response, headers = await chat_completion( - session, - os.environ["LITELLM_MASTER_KEY"], - model="fake-openai-endpoint", - messages=[ - {"role": "user", "content": f"Hello my name is ishaan@berri.ai"} - ], - guardrails=[ - "aporia-post-guard", - "aporia-pre-guard", - ], - ) - e = exc_info.value - print(e) - assert "Aporia detected and blocked PII" in str(e) - - @pytest.mark.asyncio async def test_no_llm_guard_triggered(): """ diff --git a/tests/otel_tests/test_rerank.py b/tests/otel_tests/test_rerank.py deleted file mode 100644 index 15031ef0532..00000000000 --- a/tests/otel_tests/test_rerank.py +++ /dev/null @@ -1,66 +0,0 @@ -import os -import pytest -import asyncio -import aiohttp, openai -from openai import OpenAI, AsyncOpenAI -from typing import Optional, List, Union -from litellm._uuid import uuid - - -async def make_rerank_curl_request( - session, - key, - query, - documents, - model="rerank-english-v3.0", - top_n=3, -): - url = "http://0.0.0.0:4000/rerank" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - - data = { - "model": model, - "query": query, - "documents": documents, - "top_n": top_n, - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - if status != 200: - raise Exception(response_text) - - return await response.json() - - -@pytest.mark.asyncio -async def test_basic_rerank_on_proxy(): - """ - Test litellm.rerank() on proxy - - This SHOULD NOT call the pass through endpoints :) - """ - async with aiohttp.ClientSession() as session: - docs = [ - "Carson City is the capital city of the American state of Nevada.", - "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", - "Washington, D.C. is the capital of the United States.", - "Capital punishment has existed in the United States since before it was a country.", - ] - - try: - response = await make_rerank_curl_request( - session, - os.environ["LITELLM_MASTER_KEY"], - query="What is the capital of the United States?", - documents=docs, - ) - print("response=", response) - except Exception as e: - print(e) - pytest.fail("Rerank request failed") diff --git a/tests/pass_through_tests/test_anthropic_passthrough_python_sdkpy b/tests/pass_through_tests/test_anthropic_passthrough_python_sdkpy deleted file mode 100644 index 3321735e63c..00000000000 --- a/tests/pass_through_tests/test_anthropic_passthrough_python_sdkpy +++ /dev/null @@ -1,39 +0,0 @@ -""" -This test ensures that the proxy can passthrough anthropic requests -""" - -import pytest -import anthropic -import os - -client = anthropic.Anthropic( - base_url="http://0.0.0.0:4000/anthropic", api_key=os.environ["LITELLM_MASTER_KEY"] -) - - -def test_anthropic_basic_completion(): - print("making basic completion request to anthropic passthrough") - response = client.messages.create( - model="claude-sonnet-4-5-20250929", - max_tokens=1024, - messages=[{"role": "user", "content": "Say 'hello test' and nothing else"}], - ) - print(response) - - -def test_anthropic_streaming(): - print("making streaming request to anthropic passthrough") - collected_output = [] - - with client.messages.stream( - max_tokens=10, - messages=[ - {"role": "user", "content": "Say 'hello stream test' and nothing else"} - ], - model="claude-sonnet-4-5-20250929", - ) as stream: - for text in stream.text_stream: - collected_output.append(text) - - full_response = "".join(collected_output) - print(full_response) diff --git a/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py b/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py index 147762a19a4..40eda77949d 100644 --- a/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py @@ -94,36 +94,6 @@ class BaseAnthropicMessagesTest: print(f"Non-streaming response: {json.dumps(response, indent=2, default=str)}") return response - @pytest.mark.asyncio - async def test_streaming_base(self): - """Base test for streaming requests""" - request_params = self.model_config - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Prepare call arguments - call_args = { - "messages": messages, - "max_tokens": 100, - "stream": True, - "client": AsyncHTTPHandler(), - } - - # Add any additional config from subclass - call_args.update(request_params) - - # Call the handler - response = await litellm.anthropic.messages.acreate(**call_args) - - collected_chunks = [] - if isinstance(response, AsyncIterator): - async for chunk in response: - print("chunk=", chunk) - collected_chunks.append(chunk) - - print("collected_chunks=", collected_chunks) - return collected_chunks - @pytest.mark.asyncio async def test_response_format_consistency(self): """ diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index b552e02a11a..0f0bb8f091e 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -68,7 +68,6 @@ class TestAnthropicDirectAPI(BaseAnthropicMessagesTest): """Tests for direct Anthropic API calls""" test_non_streaming_base = None - test_streaming_base = None @property def model_config(self) -> Dict[str, Any]: @@ -88,8 +87,6 @@ class TestAnthropicDirectAPI(BaseAnthropicMessagesTest): class TestAnthropicBedrockAPI(BaseAnthropicMessagesTest): """Tests for Anthropic via Bedrock""" - test_streaming_base = None - @property def model_config(self) -> Dict[str, Any]: return { @@ -107,8 +104,6 @@ class TestAnthropicBedrockAPI(BaseAnthropicMessagesTest): class TestAnthropicOpenAIAPI(BaseAnthropicMessagesTest): """Tests for OpenAI via Anthropic messages interface""" - test_streaming_base = None - @property def model_config(self) -> Dict[str, Any]: return { diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 87038a388c7..c2ed526e4c7 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -365,40 +365,6 @@ async def test_callback_observing_stamp_before_pre_header_increment_fails_leaves assert await router.get_model_group_usage("gpt-5-mini") == (None, None) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def test_track_deployment_metrics(model_list): """Test if the 'track_deployment_metrics' function is working correctly""" from litellm.types.utils import ModelResponse @@ -416,18 +382,6 @@ def test_track_deployment_metrics(model_list): ) - - - - - - - - - - - - def test_pass_through_assistants_endpoint_factory(model_list): """Test if the 'pass_through_assistants_endpoint_factory' function is working correctly""" router = Router(model_list=model_list) diff --git a/tests/test_health.py b/tests/test_health.py index c553f68b559..a92c57314b1 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -66,62 +66,6 @@ async def test_health(): assert total_model_count > 0 -@pytest.mark.asyncio -async def test_health_readiness(): - """ - Check if 200 - """ - async with aiohttp.ClientSession() as session: - url = "http://0.0.0.0:4000/health/readiness" - async with session.get(url) as response: - status = response.status - response_json = await response.json() - - print(response_json) - assert "status" in response_json - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - -@pytest.mark.asyncio -async def test_health_readiness_details(): - """ - Check if authenticated readiness diagnostics expose version metadata. - """ - async with aiohttp.ClientSession() as session: - url = "http://0.0.0.0:4000/health/readiness/details" - headers = {"Authorization": "Bearer " + os.environ["LITELLM_MASTER_KEY"]} - async with session.get(url, headers=headers) as response: - status = response.status - response_json = await response.json() - - print(response_json) - assert "status" in response_json - assert "litellm_version" in response_json - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - -@pytest.mark.asyncio -async def test_health_liveliness(): - """ - Check if 200 - """ - async with aiohttp.ClientSession() as session: - url = "http://0.0.0.0:4000/health/liveliness" - async with session.get(url) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - @pytest.mark.asyncio async def test_routes(): """ diff --git a/tests/test_keys.py b/tests/test_keys.py index f445044d990..835c1f12250 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -232,20 +232,6 @@ async def delete_key(session, get_key, auth_key=os.environ["LITELLM_MASTER_KEY"] return await response.json() -@pytest.mark.asyncio -async def test_key_delete(): - """ - Delete key - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, i=0) - key = key_gen["key"] - await delete_key( - session=session, - get_key=key, - ) - - async def get_key_info(session, call_key, get_key=None): """ Make sure only models user has access to are returned @@ -381,62 +367,6 @@ async def get_spend_logs(session, request_id): return await response.json() -@pytest.mark.skip(reason="Hanging on ci/cd") -@pytest.mark.asyncio -async def test_key_info_spend_values(): - """ - Test to ensure spend is correctly calculated - - create key - - make completion call - - assert cost is expected value - """ - - async def retry_request(func, *args, _max_attempts=5, **kwargs): - for attempt in range(_max_attempts): - try: - return await func(*args, **kwargs) - except aiohttp.client_exceptions.ClientOSError as e: - if attempt + 1 == _max_attempts: - raise # re-raise the last ClientOSError if all attempts failed - print(f"Attempt {attempt+1} failed, retrying...") - - async with aiohttp.ClientSession() as session: - ## Test Spend Update ## - # completion - key_gen = await generate_key(session=session, i=0) - key = key_gen["key"] - response = await chat_completion(session=session, key=key) - await asyncio.sleep(5) - spend_logs = await retry_request( - get_spend_logs, session=session, request_id=response["id"] - ) - print(f"spend_logs: {spend_logs}") - completion_tokens = spend_logs[0]["completion_tokens"] - prompt_tokens = spend_logs[0]["prompt_tokens"] - print(f"prompt_tokens: {prompt_tokens}; completion_tokens: {completion_tokens}") - - litellm.set_verbose = True - prompt_cost, completion_cost = litellm.cost_per_token( - model="gpt-35-turbo", - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - custom_llm_provider="azure", - ) - print("prompt_cost: ", prompt_cost, "completion_cost: ", completion_cost) - response_cost = prompt_cost + completion_cost - print(f"response_cost: {response_cost}") - await asyncio.sleep(5) # allow db log to be updated - key_info = await get_key_info(session=session, get_key=key, call_key=key) - print( - f"response_cost: {response_cost}; key_info spend: {key_info['info']['spend']}" - ) - rounded_response_cost = round(response_cost, 8) - rounded_key_info_spend = round(key_info["info"]["spend"], 8) - assert ( - rounded_response_cost == rounded_key_info_spend - ), f"Expected cost= {rounded_response_cost} != Tracked Cost={rounded_key_info_spend}" - - @pytest.mark.asyncio @pytest.mark.flaky(retries=6, delay=2) @pytest.mark.skip( @@ -524,28 +454,6 @@ async def test_key_with_budgets(): assert reset_at_init_value != reset_at_new_value -@pytest.mark.skip(reason="AWS Suspended Account") -@pytest.mark.asyncio -async def test_key_info_spend_values_sagemaker(): - """ - Tests the sync streaming loop to ensure spend is correctly calculated. - - create key - - make completion call - - assert cost is expected value - """ - async with aiohttp.ClientSession() as session: - ## streaming - sagemaker - key_gen = await generate_key(session=session, i=0, models=[]) - new_key = key_gen["key"] - prompt_tokens, completion_tokens = await chat_completion_streaming( - session=session, key=new_key, model="sagemaker-completion-model" - ) - await asyncio.sleep(5) # allow db log to be updated - key_info = await get_key_info( - session=session, get_key=new_key, call_key=new_key - ) - rounded_key_info_spend = round(key_info["info"]["spend"], 8) - assert rounded_key_info_spend > 0 # assert rounded_response_cost == rounded_key_info_spend diff --git a/tests/test_spend_logs.py b/tests/test_spend_logs.py index 503db8202b7..310900c257d 100644 --- a/tests/test_spend_logs.py +++ b/tests/test_spend_logs.py @@ -101,24 +101,6 @@ async def get_spend_logs(session, request_id=None, api_key=None): return await response.json() -@pytest.mark.skip( - reason="Flaky in CI: /spend/logs?request_id=... returns 500 even after a 20s wait for the spend log to be written. Spend-log accuracy is covered by tests/unit/proxy/spend_tracking/ and the proxy_spend_accuracy_tests CircleCI job." -) -@pytest.mark.asyncio -async def test_spend_logs(): - """ - - Create key - - Make call (makes sure it's in spend logs) - - Get request id from logs - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session) - key = key_gen["key"] - response = await chat_completion(session=session, key=key) - await asyncio.sleep(20) - await get_spend_logs(session=session, request_id=response["id"]) - - async def generate_org(session: aiohttp.ClientSession) -> dict: """ Generate a new organization using the API. @@ -236,59 +218,3 @@ async def test_get_predicted_spend_logs(): assert "response" in result assert len(result["response"]) > 0 - - -@pytest.mark.skip(reason="High traffic load test, meant to be run locally") -@pytest.mark.asyncio -async def test_spend_logs_high_traffic(): - """ - - Create key - - Make 30 concurrent calls - - Get all logs for that key - - Wait 10s - - Assert it's 30 - """ - - async def retry_request(func, *args, _max_attempts=5, **kwargs): - for attempt in range(_max_attempts): - try: - return await func(*args, **kwargs) - except ( - aiohttp.client_exceptions.ClientOSError, - aiohttp.client_exceptions.ServerDisconnectedError, - ) as e: - if attempt + 1 == _max_attempts: - raise # re-raise the last ClientOSError if all attempts failed - print(f"Attempt {attempt+1} failed, retrying...") - - async with aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=600) - ) as session: - start = time.time() - key_gen = await generate_key(session=session) - key = key_gen["key"] - n = 1000 - tasks = [ - retry_request( - chat_completion_high_traffic, - session=session, - key=key, - model="azure-gpt-3.5", - ) - for _ in range(n) - ] - chat_completions = await asyncio.gather(*tasks) - successful_completions = [c for c in chat_completions if c is not None] - print(f"Num successful completions: {len(successful_completions)}") - await asyncio.sleep(10) - try: - response = await retry_request(get_spend_logs, session=session, api_key=key) - print(f"response: {response}") - print(f"len responses: {len(response)}") - assert len(response) == n - print(n, time.time() - start, len(response)) - except Exception: - print(n, time.time() - start, 0) - raise Exception("it worked!") - - diff --git a/tests/test_team_members.py b/tests/test_team_members.py index 49dc848661b..72db6bfe7ae 100644 --- a/tests/test_team_members.py +++ b/tests/test_team_members.py @@ -137,122 +137,6 @@ def test_add_single_member(api_client, new_team): ), f"Team size did not increase by 1 (was {initial_size}, now {updated_size})" -@pytest.mark.skip( - reason="Flaky in CI: /team/info?team_id=... intermittently returns 404/400 mid-loop after add_team_member calls. Single-member coverage in test_add_single_member is sufficient; team-member CRUD is also covered by tests/unit/proxy/management_endpoints/." -) -def test_add_multiple_members(api_client, new_team): - """Test adding multiple members to a new team""" - # Get initial team size - initial_info = api_client.get_team_info(new_team) - initial_size = len(initial_info["team_info"]["members_with_roles"]) - - # Add 10 members - added_emails = [] - for i in range(10): - email = f"pytest_user_{uuid.uuid4().hex[:6]}@mycompany.com" - added_emails.append(email) - - logger.info(f"Adding member {i+1}/10: {email}") - api_client.add_team_member(new_team, email, "user") - - # Allow time for system to process - time.sleep(1) - - # Verify after each addition - current_info = api_client.get_team_info(new_team) - current_size = len(current_info["team_info"]["members_with_roles"]) - - # Assertions for each addition - assert verify_member_in_team( - current_info, email - ), f"Member {email} not found in team" - assert ( - current_size == initial_size + i + 1 - ), f"Team size incorrect after adding {email}" - - # Final verification - final_info = api_client.get_team_info(new_team) - final_size = len(final_info["team_info"]["members_with_roles"]) - - # Final assertions - assert ( - final_size == initial_size + 10 - ), f"Final team size incorrect (expected {initial_size + 10}, got {final_size})" - for email in added_emails: - assert verify_member_in_team( - final_info, email - ), f"Member {email} not found in final team check" - - -def test_team_info_structure(api_client, new_team): - """Test the structure of team info response""" - team_info = api_client.get_team_info(new_team) - - # Verify required fields exist - assert "team_id" in team_info - assert "team_info" in team_info - assert "members_with_roles" in team_info["team_info"] - assert "models" in team_info["team_info"] - - # Verify member structure - if team_info["team_info"]["members_with_roles"]: - member = team_info["team_info"]["members_with_roles"][0] - assert "user_id" in member - assert "role" in member - - -def test_error_handling(api_client): - """Test error handling for invalid team ID""" - with pytest.raises(requests.exceptions.HTTPError): - api_client.get_team_info("invalid-team-id") - - -@pytest.mark.skip( - reason="Flaky in CI: /team/info?team_id=... intermittently returns 404 after add_team_member calls, same race documented for test_add_multiple_members. Duplicate-prevention is covered by test_update_team_members_list_duplicate_prevention in tests/unit/proxy/management_endpoints/test_team_endpoints.py." -) -def test_duplicate_user_addition(api_client, new_team): - """Test that adding the same user twice is handled appropriately""" - # Add user first time - test_email = f"pytest_user_{uuid.uuid4().hex[:6]}@mycompany.com" - initial_response = api_client.add_team_member(new_team, test_email, "user") - - # Allow time for system to process - time.sleep(1) - - # Get team info after first addition - team_info_after_first = api_client.get_team_info(new_team) - size_after_first = len(team_info_after_first["team_info"]["members_with_roles"]) - - logger.info(f"First addition completed. Team size: {size_after_first}") - - # Attempt to add same user again - with pytest.raises(requests.exceptions.HTTPError): - api_client.add_team_member(new_team, test_email, "user") - - # Allow time for system to process - time.sleep(1) - - # Get team info after second addition attempt - team_info_after_second = api_client.get_team_info(new_team) - size_after_second = len(team_info_after_second["team_info"]["members_with_roles"]) - - # Verify team size didn't change - assert ( - size_after_second == size_after_first - ), f"Team size changed after duplicate addition (was {size_after_first}, now {size_after_second})" - - # Verify user appears exactly once - user_count = sum( - 1 - for member in team_info_after_second["team_info"]["members_with_roles"] - if member["user_id"] == test_email - ) - assert user_count == 1, f"User appears {user_count} times in team (expected 1)" - - logger.info(f"Duplicate addition attempted. Final team size: {size_after_second}") - logger.info(f"Number of times user appears in team: {user_count}") - - def test_member_deletion(api_client, new_team): """Test that member deletion works correctly and removes all instances of a user""" # Add a test user diff --git a/tests/test_users.py b/tests/test_users.py index f1e1a59f8db..55caeb592c8 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -110,16 +110,6 @@ async def test_user_info(): assert status == 403 -@pytest.mark.asyncio -async def test_user_update(): - """ - Create user - Update user access to new model - Make chat completion call - """ - pass - - @pytest.mark.skip(reason="Frequent check on ci/cd leads to read timeout issue.") @pytest.mark.asyncio async def test_users_budgets_reset(): @@ -182,38 +172,6 @@ async def chat_completion_streaming(session, key, model="gpt-4"): continue -@pytest.mark.skip(reason="Global proxy now tracked via `/global/spend/logs`") -@pytest.mark.asyncio -async def test_global_proxy_budget_update(): - """ - - Get proxy current spend - - Make chat completion call (normal) - - Assert spend increased - - Make chat completion call (streaming) - - Assert spend increased - """ - get_user = f"litellm-proxy-budget" - async with aiohttp.ClientSession() as session: - user_info = await get_user_info( - session=session, get_user=get_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - original_spend = user_info["user_info"]["spend"] - await chat_completion(session=session, key=os.environ["LITELLM_MASTER_KEY"]) - await asyncio.sleep(5) # let db update - user_info = await get_user_info( - session=session, get_user=get_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - new_spend = user_info["user_info"]["spend"] - print(f"new_spend: {new_spend}; original_spend: {original_spend}") - assert new_spend > original_spend - await chat_completion_streaming(session=session, key=os.environ["LITELLM_MASTER_KEY"]) - await asyncio.sleep(5) # let db update - user_info = await get_user_info( - session=session, get_user=get_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - new_new_spend = user_info["user_info"]["spend"] - print(f"new_spend: {new_spend}; original_spend: {original_spend}") - assert new_new_spend > new_spend import json