From a92b31a63613a64413d628ba2b3d08effb137505 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 28 Mar 2026 20:36:08 -0700 Subject: [PATCH] test: remove dead tests --- .../test_bedrock_completion.py | 75 +++--------- .../test_clarifai_completion.py | 109 ------------------ tests/local_testing/test_completion.py | 45 -------- tests/local_testing/test_streaming.py | 88 ++------------ 4 files changed, 24 insertions(+), 293 deletions(-) delete mode 100644 tests/llm_translation/test_clarifai_completion.py diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index b71e4e51877..f57c77db54d 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -217,60 +217,6 @@ def test_completion_bedrock_claude_external_client_auth(): # test_completion_bedrock_claude_external_client_auth() -@pytest.mark.skip(reason="Expired token, need to renew") -def test_completion_bedrock_claude_sts_client_auth(): - print("\ncalling bedrock claude external client auth") - import os - - aws_access_key_id = os.environ["AWS_TEMP_ACCESS_KEY_ID"] - aws_secret_access_key = os.environ["AWS_TEMP_SECRET_ACCESS_KEY"] - aws_region_name = os.environ["AWS_REGION_NAME"] - aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"] - - try: - import boto3 - - litellm.set_verbose = True - - response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", - messages=messages, - max_tokens=10, - temperature=0.1, - aws_region_name=aws_region_name, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_role_name=aws_role_name, - aws_session_name="my-test-session", - ) - - response = embedding( - model="cohere.embed-multilingual-v3", - input=["hello world"], - aws_region_name="us-east-1", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_role_name=aws_role_name, - aws_session_name="my-test-session", - ) - - response = completion( - model="gpt-3.5-turbo", - messages=messages, - aws_region_name="us-east-1", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_role_name=aws_role_name, - aws_session_name="my-test-session", - ) - # Add any assertions here to check the response - print(response) - except RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - @pytest.fixture() def bedrock_session_token_creds(): print("\ncalling oidc auto to get aws_session_token credentials") @@ -3413,7 +3359,8 @@ def test_bedrock_openai_imported_model(): print(f"URL: {url}") assert "bedrock-runtime.us-east-1.amazonaws.com" in url assert ( - "arn:aws:bedrock:us-east-1:117159858402:imported-model%2Fm4gc1mrfuddy" in url + "arn:aws:bedrock:us-east-1:117159858402:imported-model%2Fm4gc1mrfuddy" + in url ) assert "/invoke" in url @@ -3850,10 +3797,12 @@ def test_bedrock_openai_error_handling(): assert exc_info.value.status_code == 422 print("✓ Error handling works correctly") + # ============================================================================ # Nova Grounding (web_search_options) Unit Tests (Mocked) # ============================================================================ + def test_bedrock_nova_grounding_web_search_options_non_streaming(): """ Unit test for Nova grounding using web_search_options parameter (non-streaming). @@ -3907,7 +3856,9 @@ def test_bedrock_nova_grounding_web_search_options_non_streaming(): break assert system_tool_found, "systemTool with nova_grounding should be present" - print(f"✓ web_search_options correctly transformed to systemTool (non-streaming)") + print( + f"✓ web_search_options correctly transformed to systemTool (non-streaming)" + ) def test_bedrock_nova_grounding_with_function_tools(): @@ -3987,7 +3938,9 @@ def test_bedrock_nova_grounding_with_function_tools(): assert tool["systemTool"]["name"] == "nova_grounding" system_tool_found = True - assert function_tool_found, "Function tool (get_stock_price) should be present" + assert ( + function_tool_found + ), "Function tool (get_stock_price) should be present" assert system_tool_found, "systemTool (nova_grounding) should be present" print(f"✓ Both function tools and web_search_options correctly combined") @@ -4092,10 +4045,12 @@ def test_bedrock_nova_grounding_request_transformation(): mock_post.return_value = MagicMock( status_code=200, json=lambda: { - "output": {"message": {"role": "assistant", "content": [{"text": "Test"}]}}, + "output": { + "message": {"role": "assistant", "content": [{"text": "Test"}]} + }, "stopReason": "end_turn", - "usage": {"inputTokens": 10, "outputTokens": 5} - } + "usage": {"inputTokens": 10, "outputTokens": 5}, + }, ) try: diff --git a/tests/llm_translation/test_clarifai_completion.py b/tests/llm_translation/test_clarifai_completion.py deleted file mode 100644 index 5080413f2e5..00000000000 --- a/tests/llm_translation/test_clarifai_completion.py +++ /dev/null @@ -1,109 +0,0 @@ -import sys, os -import traceback -from dotenv import load_dotenv -import asyncio, logging - -load_dotenv() -import os, io - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import pytest -import litellm -from litellm import ( - embedding, - completion, - acompletion, - acreate, - completion_cost, - Timeout, - ModelResponse, -) -from litellm import RateLimitError - -# litellm.num_retries = 3 -litellm.cache = None -litellm.success_callback = [] -user_message = "Write a short poem about the sky" -messages = [{"content": user_message, "role": "user"}] - - -@pytest.fixture(autouse=True) -def reset_callbacks(): - print("\npytest fixture - resetting callbacks") - litellm.success_callback = [] - litellm._async_success_callback = [] - litellm.failure_callback = [] - litellm.callbacks = [] - - -@pytest.mark.skip(reason="Account rate limited.") -def test_completion_clarifai_claude_2_1(): - print("calling clarifai claude completion") - import os - - clarifai_pat = os.environ["CLARIFAI_API_KEY"] - - try: - response = completion( - model="clarifai/anthropic.completion.claude-2_1", - num_retries=3, - messages=messages, - max_tokens=10, - temperature=0.1, - ) - print(response) - - except RateLimitError: - pass - - except Exception as e: - pytest.fail(f"Error occured: {e}") - - -@pytest.mark.skip(reason="Account rate limited") -def test_completion_clarifai_mistral_large(): - try: - litellm.set_verbose = True - response: ModelResponse = completion( - model="clarifai/mistralai.completion.mistral-small", - messages=messages, - num_retries=3, - max_tokens=10, - temperature=0.78, - ) - # Add any assertions here to check the response - assert len(response.choices) > 0 - assert len(response.choices[0].message.content) > 0 - except RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.skip(reason="Account rate limited") -@pytest.mark.asyncio -def test_async_completion_clarifai(): - import asyncio - - litellm.set_verbose = True - - async def test_get_response(): - user_message = "Hello, how are you?" - messages = [{"content": user_message, "role": "user"}] - try: - response = await acompletion( - model="clarifai/openai.chat-completion.GPT-4", - messages=messages, - num_retries=3, - timeout=10, - api_key=os.getenv("CLARIFAI_API_KEY"), - ) - print(f"response: {response}") - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred: {e}") - - asyncio.run(test_get_response()) diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index af888e23cf1..d5da0719e0e 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -176,34 +176,6 @@ def predibase_mock_post(url, data=None, json=None, headers=None, timeout=None): return mock_response -# @pytest.mark.skip(reason="local-only test") -@pytest.mark.asyncio -async def test_completion_predibase(): - try: - litellm.set_verbose = True - - # with patch("requests.post", side_effect=predibase_mock_post): - response = await litellm.acompletion( - model="predibase/llama-3-8b-instruct", - tenant_id="c4768f95", - api_key=os.getenv("PREDIBASE_API_KEY"), - messages=[{"role": "user", "content": "who are u?"}], - max_tokens=10, - timeout=5, - ) - - print(response) - except litellm.Timeout as e: - print("got a timeout error from predibase") - pass - except litellm.ServiceUnavailableError as e: - pass - except litellm.InternalServerError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - # test_completion_predibase() @@ -3238,23 +3210,6 @@ def test_mistral_anyscale_stream(): print(chunk["choices"][0]["delta"].get("content", ""), end="") -# test_completion_anyscale_2() -# def test_completion_with_fallbacks_multiple_keys(): -# print(f"backup key 1: {os.getenv('BACKUP_OPENAI_API_KEY_1')}") -# print(f"backup key 2: {os.getenv('BACKUP_OPENAI_API_KEY_2')}") -# backup_keys = [{"api_key": os.getenv("BACKUP_OPENAI_API_KEY_1")}, {"api_key": os.getenv("BACKUP_OPENAI_API_KEY_2")}] -# try: -# api_key = "bad-key" -# response = completion( -# model="gpt-3.5-turbo", messages=messages, force_timeout=120, fallbacks=backup_keys, api_key=api_key -# ) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# error_str = traceback.format_exc() -# pytest.fail(f"Error occurred: {error_str}") - - # test_completion_with_fallbacks_multiple_keys() def test_petals(): try: diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 56f0e5fe826..97d6875f161 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -552,7 +552,6 @@ async def test_completion_predibase_streaming(sync_mode): pytest.fail(f"Error occurred: {e}") - def test_completion_azure_function_calling_stream(): try: litellm.set_verbose = False @@ -1655,80 +1654,9 @@ def test_sagemaker_weird_response(): # test_sagemaker_weird_response() -@pytest.mark.skip(reason="Move to being a mock endpoint") -@pytest.mark.asyncio -async def test_sagemaker_streaming_async(): - try: - messages = [{"role": "user", "content": "Hey, how's it going?"}] - litellm.set_verbose = True - response = await litellm.acompletion( - model="sagemaker/jumpstart-dft-hf-llm-mistral-7b-ins-20240329-150233", - model_id="huggingface-llm-mistral-7b-instruct-20240329-150233", - messages=messages, - temperature=0.2, - max_tokens=80, - aws_region_name=os.getenv("AWS_REGION_NAME_2"), - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID_2"), - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY_2"), - stream=True, - ) - # Add any assertions here to check the response - print(response) - complete_response = "" - has_finish_reason = False - # Add any assertions here to check the response - idx = 0 - async for chunk in response: - # print - chunk, finished = streaming_format_tests(idx, chunk) - has_finish_reason = finished - complete_response += chunk - if finished: - break - idx += 1 - if has_finish_reason is False: - raise Exception("finish reason not set for last chunk") - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"completion_response: {complete_response}") - except Exception as e: - pytest.fail(f"An exception occurred - {str(e)}") - - # asyncio.run(test_sagemaker_streaming_async()) -@pytest.mark.skip(reason="costly sagemaker deployment. Move to mock implementation") -def test_completion_sagemaker_stream(): - try: - response = completion( - model="sagemaker/jumpstart-dft-hf-llm-mistral-7b-ins-20240329-150233", - model_id="huggingface-llm-mistral-7b-instruct-20240329-150233", - messages=messages, - temperature=0.2, - max_tokens=80, - aws_region_name=os.getenv("AWS_REGION_NAME_2"), - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID_2"), - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY_2"), - stream=True, - ) - complete_response = "" - has_finish_reason = False - # Add any assertions here to check the response - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - has_finish_reason = finished - if finished: - break - complete_response += chunk - if has_finish_reason is False: - raise Exception("finish reason not set for last chunk") - if complete_response.strip() == "": - raise Exception("Empty response received") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - @pytest.mark.skip(reason="Account deleted by IBM.") @pytest.mark.asyncio async def test_completion_watsonx_stream(): @@ -2725,8 +2653,8 @@ def test_azure_streaming_and_function_calling(): tool_choice="auto", messages=messages, stream=True, - api_base=os.getenv("AZURE_API_BASE"), - api_key=os.getenv("AZURE_API_KEY"), + api_base=os.getenv("AZURE_AI_API_BASE"), + api_key=os.getenv("AZURE_AI_API_KEY"), api_version="2024-02-15-preview", ) # Add any assertions here to check the response @@ -2796,8 +2724,8 @@ async def test_azure_astreaming_and_function_calling(): tool_choice="auto", messages=messages, stream=True, - api_base=os.getenv("AZURE_API_BASE"), - api_key=os.getenv("AZURE_API_KEY"), + api_base=os.getenv("AZURE_AI_API_BASE"), + api_key=os.getenv("AZURE_AI_API_KEY"), api_version="2024-02-15-preview", caching=True, ) @@ -2827,8 +2755,8 @@ async def test_azure_astreaming_and_function_calling(): tool_choice="auto", messages=messages, stream=True, - api_base=os.getenv("AZURE_API_BASE"), - api_key=os.getenv("AZURE_API_KEY"), + api_base=os.getenv("AZURE_AI_API_BASE"), + api_key=os.getenv("AZURE_AI_API_KEY"), api_version="2024-02-15-preview", caching=True, ) @@ -3109,7 +3037,9 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk( print(f"expected_chunk_fail: {expected_chunk_fail}") if (loop_amount > litellm.REPEATED_STREAMING_CHUNK_LIMIT) and expected_chunk_fail: - with pytest.raises((litellm.InternalServerError, litellm.exceptions.MidStreamFallbackError)): + with pytest.raises( + (litellm.InternalServerError, litellm.exceptions.MidStreamFallbackError) + ): for chunk in response: continue else: