mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
test: delete 73 legacy tests covered by e2e, unable to fail, or dead in CI (#45195)
* test: delete 77 legacy tests covered by e2e, unable to fail, or dead in CI * test: keep router helper tests and coverage ignore list unchanged * test: keep cohere error handling tests --------- Co-authored-by: yuneng <yuneng@berri.ai>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
c38505d265
commit
0734e35024
39 changed files with 4 additions and 1595 deletions
|
|
@ -30,15 +30,6 @@ async def test_azure_health_check():
|
|||
# asyncio.run(test_azure_health_check())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_completion_health_check():
|
||||
response = await litellm.ahealth_check(
|
||||
model_params={"model": "gpt-3.5-turbo-instruct"},
|
||||
mode="completion",
|
||||
prompt="What's the weather in SF?",
|
||||
)
|
||||
print(f"response: {response}")
|
||||
return response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -333,17 +324,3 @@ async def test_timeout_does_not_cancel_other_health_checks():
|
|||
|
||||
assert "openai/fast-model" in healthy_models
|
||||
assert "openai/slow-model" in unhealthy_models
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ahealth_check_ocr():
|
||||
litellm.turn_on_debug()
|
||||
response = await litellm.ahealth_check(
|
||||
model_params={
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"api_key": os.getenv("MISTRAL_API_KEY"),
|
||||
},
|
||||
mode="ocr",
|
||||
)
|
||||
print(response)
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ from openai import OpenAI
|
|||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
|
||||
|
||||
from tests._live_test_helpers import _skip_live_prompt_caching_test # noqa: E402
|
||||
|
||||
|
||||
def _usage_format_tests(usage: litellm.Usage):
|
||||
|
|
@ -640,15 +639,6 @@ class BaseLLMChatTest(ABC):
|
|||
except litellm.InternalServerError:
|
||||
pytest.skip("Model is overloaded")
|
||||
|
||||
@pytest.mark.flaky(retries=6, delay=1)
|
||||
def test_json_response_pydantic_obj_nested_obj(self):
|
||||
litellm.set_verbose = True
|
||||
from pydantic import BaseModel
|
||||
from litellm.utils import supports_response_schema
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
@pytest.mark.flaky(retries=6, delay=1)
|
||||
def test_json_response_nested_pydantic_obj(self):
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -845,11 +835,6 @@ class BaseLLMChatTest(ABC):
|
|||
],
|
||||
}
|
||||
|
||||
@abstractmethod
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
@pytest.mark.parametrize("detail", [None, "low", "high"])
|
||||
@pytest.mark.parametrize(
|
||||
"image_url",
|
||||
|
|
@ -962,108 +947,6 @@ class BaseLLMChatTest(ABC):
|
|||
|
||||
assert response is not None
|
||||
|
||||
@pytest.mark.flaky(retries=4, delay=1)
|
||||
def test_prompt_caching(self):
|
||||
_skip_live_prompt_caching_test()
|
||||
print("test_prompt_caching")
|
||||
litellm.set_verbose = True
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
if not supports_prompt_caching(base_completion_call_args["model"], None):
|
||||
print("Model does not support prompt caching")
|
||||
pytest.skip("Model does not support prompt caching")
|
||||
|
||||
uuid_str = str(uuid.uuid4())
|
||||
messages = [
|
||||
# System Message
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement {}".format(
|
||||
uuid_str
|
||||
)
|
||||
* 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
# marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What are the key terms and conditions in this agreement?",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
|
||||
},
|
||||
# The final turn is marked with cache-control, for continuing in followups.
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What are the key terms and conditions in this agreement?",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
## call 1
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
)
|
||||
|
||||
print("response=", response)
|
||||
|
||||
initial_cost = response._hidden_params["response_cost"]
|
||||
## call 2
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
)
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
cached_cost = response._hidden_params["response_cost"]
|
||||
|
||||
assert (
|
||||
cached_cost <= initial_cost
|
||||
), "Cached cost={} should be less than initial cost={}".format(
|
||||
cached_cost, initial_cost
|
||||
)
|
||||
|
||||
_usage_format_tests(response.usage)
|
||||
|
||||
print("response=", response)
|
||||
print("response.usage=", response.usage)
|
||||
|
||||
_usage_format_tests(response.usage)
|
||||
|
||||
assert "prompt_tokens_details" in response.usage
|
||||
if response.usage.prompt_tokens_details is not None:
|
||||
assert (
|
||||
response.usage.prompt_tokens_details.cached_tokens > 0
|
||||
), f"cached_tokens={response.usage.prompt_tokens_details.cached_tokens} should be greater than 0. Got usage={response.usage}"
|
||||
except litellm.InternalServerError as e:
|
||||
print("InternalServerError", e)
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_messages(self):
|
||||
import base64
|
||||
|
|
|
|||
|
|
@ -439,17 +439,3 @@ class BaseRealtimeTest(ABC):
|
|||
|
||||
assert websocket_client.connection_successful, "Failed to establish connection"
|
||||
assert websocket_client.sent_user_message, "Failed to send user message"
|
||||
|
||||
def test_query_params_construction(self):
|
||||
"""Test that query params are constructed correctly"""
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
# Strip provider prefix from model name
|
||||
model_name = self.get_model()
|
||||
if "/" in model_name:
|
||||
model_name = model_name.split("/", 1)[1]
|
||||
|
||||
query_params: RealtimeQueryParams = {"model": model_name}
|
||||
|
||||
assert "model" in query_params
|
||||
assert query_params["model"] == model_name
|
||||
|
|
|
|||
|
|
@ -136,33 +136,3 @@ async def test_openai_realtime_direct_call_no_intent():
|
|||
), "session.created response missing session object"
|
||||
assert "id" in session_message["session"], "Session object missing id field"
|
||||
assert "model" in session_message["session"], "Session object missing model field"
|
||||
|
||||
|
||||
def test_realtime_query_params_construction():
|
||||
"""
|
||||
Test that query params are constructed correctly by the proxy server logic
|
||||
"""
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
# Test case 1: intent is None (should not be included)
|
||||
model = "gpt-4o-realtime-preview"
|
||||
intent = None
|
||||
|
||||
query_params: RealtimeQueryParams = {"model": model}
|
||||
if intent is not None:
|
||||
query_params["intent"] = intent
|
||||
|
||||
assert "model" in query_params
|
||||
assert query_params["model"] == model
|
||||
assert "intent" not in query_params
|
||||
|
||||
# Test case 2: intent is provided (should be included)
|
||||
intent = "chat"
|
||||
query_params2: RealtimeQueryParams = {"model": model}
|
||||
if intent is not None:
|
||||
query_params2["intent"] = intent
|
||||
|
||||
assert "model" in query_params2
|
||||
assert query_params2["model"] == model
|
||||
assert "intent" in query_params2
|
||||
assert query_params2["intent"] == intent
|
||||
|
|
|
|||
|
|
@ -315,15 +315,6 @@ class TestAnthropicCompletion(BaseLLMChatTest, BaseAnthropicChatTest):
|
|||
"thinking": {"type": "enabled", "budget_tokens": 16000},
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_anthropic_tool_invoke,
|
||||
)
|
||||
|
||||
result = convert_to_anthropic_tool_invoke([tool_call_no_arguments])
|
||||
print(result)
|
||||
|
||||
def test_tool_call_and_json_response_format(self):
|
||||
"""
|
||||
Test that the tool call and JSON response format is supported by the LLM API
|
||||
|
|
|
|||
|
|
@ -31,16 +31,9 @@ class TestAzureOpenAIO3Mini(BaseOSeriesModelsTest, BaseLLMChatTest):
|
|||
api_version="2024-02-15-preview",
|
||||
)
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_basic_tool_calling(self):
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""Temporary override. o1 prompt caching is not working."""
|
||||
pass
|
||||
|
||||
|
||||
class TestAzureOpenAIO3(BaseOSeriesModelsTest):
|
||||
|
|
|
|||
|
|
@ -709,16 +709,6 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest):
|
|||
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""
|
||||
Remove override once we have access to Bedrock prompt caching
|
||||
"""
|
||||
pass
|
||||
|
||||
def test_completion_cost(self):
|
||||
"""
|
||||
Test if region models info is correctly used for cost calculation. Using the base model info for cost calculation.
|
||||
|
|
@ -770,9 +760,6 @@ class TestBedrockConverseChatNormal(BaseLLMChatTest):
|
|||
"aws_region_name": "us-east-1",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
|
||||
class TestBedrockConverseNovaTestSuite(BaseLLMChatTest):
|
||||
|
|
@ -789,14 +776,7 @@ class TestBedrockConverseNovaTestSuite(BaseLLMChatTest):
|
|||
"aws_region_name": "us-east-1",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""
|
||||
TODO: Ensure this test passes our base llm test suite
|
||||
"""
|
||||
|
||||
|
||||
class TestBedrockRerank(BaseLLMRerankTest):
|
||||
|
|
|
|||
|
|
@ -9,10 +9,6 @@ class TestBedrockGPTOSS(BaseLLMChatTest):
|
|||
"model": "bedrock/converse/openai.gpt-oss-20b-1:0",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_function_calling_with_tool_response(self):
|
||||
"""Bedrock GPT-OSS intermittently emits truncated toolUse.input deltas on
|
||||
the live endpoint, which makes the inherited live integration test flaky.
|
||||
|
|
@ -23,13 +19,6 @@ class TestBedrockGPTOSS(BaseLLMChatTest):
|
|||
"""
|
||||
pass
|
||||
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""
|
||||
Remove override once we have access to Bedrock prompt caching
|
||||
"""
|
||||
pass
|
||||
|
||||
async def test_completion_cost(self):
|
||||
"""
|
||||
Bedrock GPT-OSS models are flaky and occasionally report 0 token counts in api response
|
||||
|
|
|
|||
|
|
@ -24,10 +24,6 @@ class TestBedrockInvokeClaudeJson(BaseLLMChatTest):
|
|||
"model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_url, detail",
|
||||
[
|
||||
|
|
@ -54,10 +50,6 @@ class TestBedrockInvokeNovaJson(BaseLLMChatTest):
|
|||
"model": "bedrock/invoke/us.amazon.nova-micro-v1:0",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def skip_non_json_tests(self, request):
|
||||
if not "json" in request.function.__name__.lower():
|
||||
|
|
|
|||
|
|
@ -9,9 +9,6 @@ class TestBedrockTestSuite(BaseLLMChatTest):
|
|||
test_empty_tools = None
|
||||
test_function_calling_with_tool_response = None
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
pass
|
||||
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
litellm.turn_on_debug()
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -38,22 +38,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest):
|
|||
"model": "bedrock/invoke/moonshot.kimi-k2-thinking",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly."""
|
||||
pass
|
||||
|
||||
|
||||
class TestBedrockMoonshotToolCalling:
|
||||
"""Unit tests for tool calling functionality."""
|
||||
|
||||
def test_tool_response_message_format(self):
|
||||
"""Test that tool response messages are formatted correctly."""
|
||||
tool_response_message = {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_123",
|
||||
"content": json.dumps({"temperature": 72, "condition": "sunny"}),
|
||||
}
|
||||
|
||||
assert tool_response_message["role"] == "tool"
|
||||
assert "tool_call_id" in tool_response_message
|
||||
assert "content" in tool_response_message
|
||||
|
|
|
|||
|
|
@ -25,15 +25,7 @@ class TestBedrockNovaJson(BaseLLMChatTest):
|
|||
def test_json_response_nested_json_schema(self):
|
||||
pass
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""
|
||||
Remove override once we have access to Bedrock prompt caching
|
||||
"""
|
||||
pass
|
||||
|
||||
# @pytest.fixture(autouse=True)
|
||||
# def skip_non_json_tests(self, request):
|
||||
|
|
|
|||
|
|
@ -260,23 +260,6 @@ class BaseEvalsAPITest(ABC):
|
|||
assert response.name == updated_name
|
||||
print(f"Updated eval: {response}")
|
||||
|
||||
def test_delete_eval(self):
|
||||
"""
|
||||
Test deleting an evaluation.
|
||||
|
||||
Real delete coverage now lives in the ``managed_eval`` fixture
|
||||
teardown and in ``test_create_eval``'s ``finally`` block, so
|
||||
this stays a no-op skip rather than creating a fresh resource
|
||||
just to delete it.
|
||||
"""
|
||||
custom_llm_provider = self.get_custom_llm_provider()
|
||||
api_key = self.get_api_key()
|
||||
api_base = self.get_api_base()
|
||||
|
||||
if not api_key:
|
||||
pytest.skip(f"No API key provided for {custom_llm_provider}")
|
||||
|
||||
pytest.skip("Delete is exercised via managed_eval fixture teardown.")
|
||||
|
||||
|
||||
class TestOpenAIEvalsAPI(BaseEvalsAPITest):
|
||||
|
|
|
|||
|
|
@ -19,9 +19,5 @@ class TestGroq(BaseLLMChatTest):
|
|||
"model": "groq/openai/gpt-oss-120b",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_tool_call_with_empty_enum_property(self):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -29,7 +29,3 @@ class TestMistralCompletion(BaseLLMChatTest):
|
|||
def get_base_completion_call_args(self) -> dict:
|
||||
litellm.set_verbose = True
|
||||
return {"model": "mistral/mistral-medium-latest"}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -215,16 +215,6 @@ class TestOpenAIChatCompletion(BaseLLMChatTest):
|
|||
def get_base_completion_call_args(self) -> dict:
|
||||
return {"model": "gpt-4o-mini"}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""
|
||||
Works locally but CI/CD is failing this test. Temporary skip to push out a new release.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["o1", "o3-mini"])
|
||||
def test_o1_parallel_tool_calls(model):
|
||||
|
|
|
|||
|
|
@ -24,13 +24,7 @@ class TestOpenAIO1(BaseOSeriesModelsTest, BaseLLMChatTest):
|
|||
|
||||
return OpenAI(api_key="fake-api-key")
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""Temporary override. o1 prompt caching is not working."""
|
||||
pass
|
||||
|
||||
|
||||
class TestOpenAIO3(BaseOSeriesModelsTest, BaseLLMChatTest):
|
||||
|
|
@ -47,13 +41,7 @@ class TestOpenAIO3(BaseOSeriesModelsTest, BaseLLMChatTest):
|
|||
|
||||
return OpenAI(api_key="fake-api-key")
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""Override, as o3 prompt caching is flaky"""
|
||||
pass
|
||||
|
||||
|
||||
def test_o3_reasoning_effort():
|
||||
|
|
|
|||
|
|
@ -41,16 +41,6 @@ class TestRouterLLMTranslation(BaseLLMChatTest):
|
|||
def get_base_completion_call_args(self) -> dict:
|
||||
return {"model": "gpt-4o-mini"}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
def test_prompt_caching(self):
|
||||
"""
|
||||
Works locally but CI/CD is failing this test. Temporary skip to push out a new release.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def test_router_azure_acompletion():
|
||||
# [PROD TEST CASE]
|
||||
|
|
|
|||
|
|
@ -27,7 +27,3 @@ class TestTogetherAI(BaseLLMChatTest):
|
|||
def get_base_completion_call_args(self) -> dict:
|
||||
litellm.set_verbose = True
|
||||
return {"model": cheapest_together_chat_model(function_calling=True, response_schema=True)}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -53,10 +53,6 @@ class TestXAIChat(BaseLLMChatTest):
|
|||
"model": "xai/grok-3-mini-beta",
|
||||
}
|
||||
|
||||
def test_tool_call_no_arguments(self, tool_call_no_arguments):
|
||||
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
|
||||
pass
|
||||
|
||||
test_web_search = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,47 +9,10 @@ litellm.success_callback = ["lunary"]
|
|||
litellm.set_verbose = True
|
||||
|
||||
|
||||
def test_lunary_logging():
|
||||
try:
|
||||
response = completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "what llm are u"}],
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
user="test-user",
|
||||
)
|
||||
print(response)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
|
||||
def test_lunary_template():
|
||||
import lunary
|
||||
|
||||
try:
|
||||
template = lunary.render_template("test-template", {"question": "Hello!"})
|
||||
response = completion(**template)
|
||||
print(response)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
|
||||
def test_lunary_logging_with_metadata():
|
||||
try:
|
||||
response = completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "what llm are u"}],
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
metadata={
|
||||
"run_name": "litellmRUN",
|
||||
"project_name": "litellm-completion",
|
||||
"tags": ["tag1", "tag2"],
|
||||
},
|
||||
)
|
||||
print(response)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
|
||||
def test_lunary_with_tools():
|
||||
|
|
@ -93,22 +56,3 @@ def test_lunary_with_tools():
|
|||
assert response.choices[0].message.tool_calls
|
||||
assert all(call.function.name == "get_current_weather" for call in response.choices[0].message.tool_calls)
|
||||
print("\nLLM Response:\n", response.choices[0].message)
|
||||
|
||||
|
||||
def test_lunary_logging_with_streaming_and_metadata():
|
||||
try:
|
||||
response = completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "what llm are u"}],
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
metadata={
|
||||
"run_name": "litellmRUN",
|
||||
"project_name": "litellm-completion",
|
||||
},
|
||||
stream=True,
|
||||
)
|
||||
for chunk in response:
|
||||
continue
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
|
|
|||
|
|
@ -450,50 +450,6 @@ def test_function_calling():
|
|||
# test_acompletion_on_router()
|
||||
|
||||
|
||||
def test_function_calling_on_router():
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
},
|
||||
]
|
||||
function1 = [
|
||||
{
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
}
|
||||
]
|
||||
router = Router(
|
||||
model_list=model_list,
|
||||
redis_host=os.getenv("REDIS_HOST"),
|
||||
redis_password=os.getenv("REDIS_PASSWORD"),
|
||||
redis_port=os.getenv("REDIS_PORT"),
|
||||
)
|
||||
messages = [{"role": "user", "content": "what's the weather in boston"}]
|
||||
response = router.completion(
|
||||
model="gpt-3.5-turbo", messages=messages, functions=function1
|
||||
)
|
||||
print(f"final returned response: {response}")
|
||||
router.reset()
|
||||
assert isinstance(response["choices"][0]["message"]["function_call"], dict)
|
||||
except Exception as e:
|
||||
print(f"An exception occurred: {e}")
|
||||
|
||||
|
||||
# test_function_calling_on_router()
|
||||
|
|
|
|||
|
|
@ -54,85 +54,6 @@ kwargs = {
|
|||
"messages": [{"role": "user", "content": "Hey, how's it going?"}],
|
||||
}
|
||||
|
||||
def test_sync_fallbacks():
|
||||
try:
|
||||
model_list = [
|
||||
{ # list of model deployments
|
||||
"model_name": "azure/gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": "bad-key",
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800,
|
||||
},
|
||||
{ # list of model deployments
|
||||
"model_name": "azure/gpt-3.5-turbo-context-fallback", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": os.getenv("AZURE_AI_API_KEY"),
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800,
|
||||
},
|
||||
{
|
||||
"model_name": "azure/gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/chatgpt-functioncalling",
|
||||
"api_key": "bad-key",
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800,
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "gpt-3.5-turbo",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
"tpm": 1000000,
|
||||
"rpm": 9000,
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo-16k", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "gpt-3.5-turbo-16k",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
"tpm": 1000000,
|
||||
"rpm": 9000,
|
||||
},
|
||||
]
|
||||
|
||||
litellm.set_verbose = True
|
||||
customHandler = MyCustomHandler()
|
||||
litellm.callbacks = [customHandler]
|
||||
router = Router(
|
||||
model_list=model_list,
|
||||
fallbacks=[{"azure/gpt-3.5-turbo": ["gpt-3.5-turbo"]}],
|
||||
context_window_fallbacks=[
|
||||
{"azure/gpt-3.5-turbo-context-fallback": ["gpt-3.5-turbo-16k"]},
|
||||
{"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]},
|
||||
],
|
||||
set_verbose=False,
|
||||
)
|
||||
response = router.completion(**kwargs)
|
||||
print(f"response: {response}")
|
||||
time.sleep(0.05) # allow a delay as success_callbacks are on a separate thread
|
||||
assert (
|
||||
customHandler.previous_models == 3
|
||||
) # 1 init call + 2 retries (fallback not counted as previous)
|
||||
|
||||
print("Passed ! Test router_fallbacks: test_sync_fallbacks()")
|
||||
router.reset()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
# test_sync_fallbacks()
|
||||
|
||||
|
|
@ -489,83 +410,6 @@ async def test_dynamic_fallbacks_async():
|
|||
|
||||
# asyncio.run(test_dynamic_fallbacks_async())
|
||||
|
||||
def test_sync_fallbacks_streaming():
|
||||
try:
|
||||
model_list = [
|
||||
{ # list of model deployments
|
||||
"model_name": "azure/gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": "bad-key",
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800,
|
||||
},
|
||||
{ # list of model deployments
|
||||
"model_name": "azure/gpt-3.5-turbo-context-fallback", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": os.getenv("AZURE_AI_API_KEY"),
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800,
|
||||
},
|
||||
{
|
||||
"model_name": "azure/gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/chatgpt-functioncalling",
|
||||
"api_key": "bad-key",
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 240000,
|
||||
"rpm": 1800,
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "gpt-3.5-turbo",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
"tpm": 1000000,
|
||||
"rpm": 9000,
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo-16k", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "gpt-3.5-turbo-16k",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
"tpm": 1000000,
|
||||
"rpm": 9000,
|
||||
},
|
||||
]
|
||||
|
||||
litellm.set_verbose = True
|
||||
customHandler = MyCustomHandler()
|
||||
litellm.callbacks = [customHandler]
|
||||
router = Router(
|
||||
model_list=model_list,
|
||||
fallbacks=[{"azure/gpt-3.5-turbo": ["gpt-3.5-turbo"]}],
|
||||
context_window_fallbacks=[
|
||||
{"azure/gpt-3.5-turbo-context-fallback": ["gpt-3.5-turbo-16k"]},
|
||||
{"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]},
|
||||
],
|
||||
set_verbose=False,
|
||||
)
|
||||
response = router.completion(**kwargs, stream=True)
|
||||
print(f"response: {response}")
|
||||
time.sleep(0.05) # allow a delay as success_callbacks are on a separate thread
|
||||
assert customHandler.previous_models == 1 # 0 retries, 1 fallback
|
||||
|
||||
print("Passed ! Test router_fallbacks: test_sync_fallbacks()")
|
||||
router.reset()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_fallbacks_max_retries_per_request():
|
||||
|
|
@ -777,94 +621,6 @@ def test_ausage_based_routing_fallbacks():
|
|||
except Exception as e:
|
||||
pytest.fail(f"An exception occurred {e}")
|
||||
|
||||
def test_custom_cooldown_times():
|
||||
try:
|
||||
# set, custom_cooldown. Failed model in cooldown_models, after custom_cooldown, the failed model is no longer in cooldown_models
|
||||
|
||||
model_list = [
|
||||
{ # list of model deployments
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": "bad-key",
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 24000000,
|
||||
},
|
||||
{ # list of model deployments
|
||||
"model_name": "gpt-3.5-turbo", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": os.getenv("AZURE_AI_API_KEY"),
|
||||
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
},
|
||||
"tpm": 1,
|
||||
},
|
||||
]
|
||||
|
||||
litellm.set_verbose = False
|
||||
|
||||
router = Router(
|
||||
model_list=model_list,
|
||||
set_verbose=True,
|
||||
debug_level="INFO",
|
||||
cooldown_time=0.1,
|
||||
redis_host=os.getenv("REDIS_HOST"),
|
||||
redis_password=os.getenv("REDIS_PASSWORD"),
|
||||
redis_port=int(os.getenv("REDIS_PORT")),
|
||||
)
|
||||
|
||||
# make a request - expect it to fail
|
||||
try:
|
||||
response = router.completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[
|
||||
{
|
||||
"content": "Tell me a joke.",
|
||||
"role": "user",
|
||||
}
|
||||
],
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# expect 1 model to be in cooldown models
|
||||
cooldown_deployments = router._get_cooldown_deployments()
|
||||
print("cooldown_deployments after failed call: ", cooldown_deployments)
|
||||
assert (
|
||||
len(cooldown_deployments) == 1
|
||||
), "Expected 1 model to be in cooldown models"
|
||||
|
||||
selected_cooldown_model = cooldown_deployments[0]
|
||||
|
||||
# wait for 1/2 of cooldown time
|
||||
time.sleep(router.cooldown_time / 2)
|
||||
|
||||
# expect cooldown model to still be in cooldown models
|
||||
cooldown_deployments = router._get_cooldown_deployments()
|
||||
print(
|
||||
"cooldown_deployments after waiting 1/2 of cooldown: ", cooldown_deployments
|
||||
)
|
||||
assert (
|
||||
len(cooldown_deployments) == 1
|
||||
), "Expected 1 model to be in cooldown models"
|
||||
|
||||
# wait for 1/2 of cooldown time again, now we've waited for full cooldown
|
||||
time.sleep(router.cooldown_time / 2)
|
||||
|
||||
# expect cooldown model to be removed from cooldown models
|
||||
cooldown_deployments = router._get_cooldown_deployments()
|
||||
print(
|
||||
"cooldown_deployments after waiting cooldown time: ", cooldown_deployments
|
||||
)
|
||||
assert (
|
||||
len(cooldown_deployments) == 0
|
||||
), "Expected 0 models to be in cooldown models"
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -818,20 +818,6 @@ def test_openai_chat_completion_call():
|
|||
print(f"complete response: {complete_response}")
|
||||
|
||||
|
||||
def test_openai_chat_completion_complete_response_call():
|
||||
try:
|
||||
complete_response = completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
complete_response=True,
|
||||
)
|
||||
print(f"complete response: {complete_response}")
|
||||
except Exception:
|
||||
print(f"error occurred: {traceback.format_exc()}")
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
|
|
@ -925,90 +911,14 @@ def test_openai_stream_options_call_text_completion() -> None:
|
|||
assert any(chunk.choices[0].text for chunk in chunks)
|
||||
|
||||
|
||||
def test_openai_text_completion_call():
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
response = completion(
|
||||
model="gpt-3.5-turbo-instruct", messages=messages, stream=True
|
||||
)
|
||||
complete_response = ""
|
||||
start_time = time.time()
|
||||
for idx, chunk in enumerate(response):
|
||||
chunk, finished = streaming_format_tests(idx, chunk)
|
||||
print(f"chunk: {chunk}")
|
||||
complete_response += chunk
|
||||
if finished:
|
||||
break
|
||||
# print(f'complete_chunk: {complete_response}')
|
||||
if complete_response.strip() == "":
|
||||
raise Exception("Empty response received")
|
||||
print(f"complete response: {complete_response}")
|
||||
except Exception:
|
||||
print(f"error occurred: {traceback.format_exc()}")
|
||||
pass
|
||||
|
||||
|
||||
# # test on together ai completion call - starcoder
|
||||
def test_together_ai_completion_call_mistral():
|
||||
try:
|
||||
litellm.set_verbose = False
|
||||
start_time = time.time()
|
||||
response = completion(
|
||||
model="together_ai/mistralai/Mistral-7B-Instruct-v0.2",
|
||||
messages=messages,
|
||||
logger_fn=logger_fn,
|
||||
stream=True,
|
||||
)
|
||||
complete_response = ""
|
||||
print(f"returned response object: {response}")
|
||||
has_finish_reason = False
|
||||
for idx, chunk in enumerate(response):
|
||||
chunk, finished = streaming_format_tests(idx, chunk)
|
||||
has_finish_reason = finished
|
||||
if finished:
|
||||
break
|
||||
complete_response += chunk
|
||||
if has_finish_reason is False:
|
||||
raise Exception("Finish reason not set for last chunk")
|
||||
if complete_response == "":
|
||||
raise Exception("Empty response received")
|
||||
print(f"complete response: {complete_response}")
|
||||
except Exception:
|
||||
print(f"error occurred: {traceback.format_exc()}")
|
||||
pass
|
||||
|
||||
|
||||
# # test on together ai completion call - starcoder
|
||||
|
||||
|
||||
def test_together_ai_completion_call_starcoder_bad_key():
|
||||
try:
|
||||
api_key = "bad-key"
|
||||
start_time = time.time()
|
||||
response = completion(
|
||||
model="together_ai/bigcode/starcoder",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
api_key=api_key,
|
||||
)
|
||||
complete_response = ""
|
||||
has_finish_reason = False
|
||||
for idx, chunk in enumerate(response):
|
||||
chunk, finished = streaming_format_tests(idx, chunk)
|
||||
has_finish_reason = finished
|
||||
if finished:
|
||||
break
|
||||
complete_response += chunk
|
||||
if has_finish_reason is False:
|
||||
raise Exception("Finish reason not set for last chunk")
|
||||
if complete_response == "":
|
||||
raise Exception("Empty response received")
|
||||
print(f"complete response: {complete_response}")
|
||||
except BadRequestError as e:
|
||||
pass
|
||||
except Exception:
|
||||
print(f"error occurred: {traceback.format_exc()}")
|
||||
pass
|
||||
# # test on together ai completion call - starcoder
|
||||
|
||||
|
||||
|
||||
|
||||
#### Test Function calling + streaming ####
|
||||
|
|
|
|||
|
|
@ -2850,25 +2850,6 @@ def test_completion_text_003_prompt_array():
|
|||
# asyncio.run(test_text_completion_async_stream())
|
||||
|
||||
|
||||
def test_async_text_completion():
|
||||
litellm.set_verbose = True
|
||||
print("test_async_text_completion")
|
||||
|
||||
async def test_get_response():
|
||||
try:
|
||||
response = await litellm.atext_completion(
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
prompt="good morning",
|
||||
stream=False,
|
||||
max_tokens=10,
|
||||
)
|
||||
print(f"response: {response}")
|
||||
except litellm.Timeout as e:
|
||||
print(e)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
asyncio.run(test_get_response())
|
||||
|
||||
|
||||
# test_async_text_completion()
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import litellm
|
|||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
# test_example.py
|
||||
from abc import ABC, abstractmethod
|
||||
|
|
@ -84,12 +83,3 @@ class BaseLoggingCallbackTest(ABC):
|
|||
),
|
||||
service_tier=None,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def test_parallel_tool_calls(self, mock_response_obj: ModelResponse):
|
||||
"""
|
||||
Check if parallel tool calls are correctly logged by Logging callback
|
||||
|
||||
Relevant issue - https://github.com/BerriAI/litellm/issues/6677
|
||||
"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
"""
|
||||
Test the DataDogLLMObsLogger
|
||||
"""
|
||||
|
||||
import io
|
||||
|
||||
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import completion
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
|
||||
from datetime import datetime, timedelta
|
||||
from litellm.types.integrations.datadog_llm_obs import *
|
||||
from litellm.types.utils import (
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingModelInformation,
|
||||
StandardLoggingMetadata,
|
||||
StandardLoggingHiddenParams,
|
||||
)
|
||||
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def create_standard_logging_payload() -> StandardLoggingPayload:
|
||||
return StandardLoggingPayload(
|
||||
id="test_id",
|
||||
call_type="completion",
|
||||
response_cost=0.1,
|
||||
response_cost_failure_debug_info=None,
|
||||
status="success",
|
||||
total_tokens=30,
|
||||
prompt_tokens=20,
|
||||
completion_tokens=10,
|
||||
startTime=1234567890.0,
|
||||
endTime=1234567891.0,
|
||||
completionStartTime=1234567890.5,
|
||||
model_map_information=StandardLoggingModelInformation(
|
||||
model_map_key="gpt-5-mini", model_map_value=None
|
||||
),
|
||||
model="gpt-5-mini",
|
||||
model_id="model-123",
|
||||
model_group="openai-gpt",
|
||||
api_base="https://api.openai.com",
|
||||
metadata=StandardLoggingMetadata(
|
||||
user_api_key_hash="test_hash",
|
||||
user_api_key_org_id=None,
|
||||
user_api_key_alias="test_alias",
|
||||
user_api_key_team_id="test_team",
|
||||
user_api_key_user_id="test_user",
|
||||
user_api_key_team_alias="test_team_alias",
|
||||
spend_logs_metadata=None,
|
||||
requester_ip_address="127.0.0.1",
|
||||
requester_metadata=None,
|
||||
),
|
||||
cache_hit=False,
|
||||
cache_key=None,
|
||||
saved_cache_cost=0.0,
|
||||
request_tags=[],
|
||||
end_user=None,
|
||||
requester_ip_address="127.0.0.1",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
response={"choices": [{"message": {"content": "Hi there!"}}]},
|
||||
error_str=None,
|
||||
model_parameters={"stream": True},
|
||||
hidden_params=StandardLoggingHiddenParams(
|
||||
model_id="model-123",
|
||||
cache_key=None,
|
||||
api_base="https://api.openai.com",
|
||||
response_cost="0.1",
|
||||
additional_headers=None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_datadog_llm_obs_logging():
|
||||
datadog_llm_obs_logger = DataDogLLMObsLogger()
|
||||
litellm.callbacks = [datadog_llm_obs_logger]
|
||||
litellm.set_verbose = True
|
||||
|
||||
for _ in range(2):
|
||||
response = await litellm.acompletion(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "Hello testing dd llm obs!"}],
|
||||
mock_response="hi",
|
||||
)
|
||||
|
||||
print(response)
|
||||
|
||||
await asyncio.sleep(6)
|
||||
|
|
@ -9,7 +9,7 @@ from typing import Optional, List, Union
|
|||
from test_openai_files_endpoints import upload_file, delete_file
|
||||
import sys
|
||||
import time
|
||||
from unittest.mock import patch, MagicMock, AsyncMock
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
BASE_URL = "http://localhost:4000" # Replace with your actual base URL
|
||||
|
|
@ -139,81 +139,6 @@ def test_vertex_batches_endpoint():
|
|||
pass
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Local only test to verify if things work well")
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_batches_with_target_model_names():
|
||||
"""
|
||||
Unit test to verify that target_model_names query parameter is properly handled
|
||||
in the list_batches endpoint
|
||||
"""
|
||||
|
||||
# Test data
|
||||
target_model_names = "gpt-5.5,gpt-5-mini"
|
||||
expected_model = "gpt-5.5" # Should use the first model from the comma-separated list
|
||||
|
||||
# Mock response for list_batches
|
||||
mock_batch_response = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "batch_abc123",
|
||||
"object": "batch",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"status": "validating",
|
||||
"input_file_id": "file-abc123",
|
||||
"completion_window": "24h",
|
||||
"created_at": 1711471533,
|
||||
"metadata": {},
|
||||
}
|
||||
],
|
||||
"first_id": "batch_abc123",
|
||||
"last_id": "batch_abc123",
|
||||
"has_more": False,
|
||||
}
|
||||
|
||||
# Mock the request and FastAPI dependencies
|
||||
mock_request = MagicMock()
|
||||
mock_request.method = "GET"
|
||||
mock_request.url.query = f"target_model_names={target_model_names}&limit=10"
|
||||
|
||||
mock_fastapi_response = MagicMock()
|
||||
mock_user_api_key_dict = MagicMock()
|
||||
|
||||
# Mock _read_request_body to return our target_model_names
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.batches_endpoints.endpoints._read_request_body"
|
||||
) as mock_read_body,
|
||||
patch("litellm.proxy.proxy_server.llm_router") as mock_router,
|
||||
):
|
||||
|
||||
mock_read_body.return_value = {"target_model_names": target_model_names}
|
||||
mock_router.alist_batches = AsyncMock(return_value=mock_batch_response)
|
||||
|
||||
# Import and call the function directly
|
||||
from litellm.proxy.batches_endpoints.endpoints import list_batches
|
||||
|
||||
response = await list_batches(
|
||||
request=mock_request,
|
||||
fastapi_response=mock_fastapi_response,
|
||||
target_model_names=target_model_names,
|
||||
limit=10,
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
)
|
||||
|
||||
# Verify that router.alist_batches was called with the correct model
|
||||
mock_router.alist_batches.assert_called_once()
|
||||
call_args = mock_router.alist_batches.call_args
|
||||
|
||||
# Check that the model parameter was set to the first model in the list
|
||||
assert call_args.kwargs["model"] == expected_model
|
||||
assert call_args.kwargs["limit"] == 10
|
||||
|
||||
# Verify the response structure
|
||||
assert response["object"] == "list"
|
||||
assert len(response["data"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_status_sync_from_provider_to_database():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -72,62 +72,6 @@ async def generate_key(
|
|||
return await response.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip(reason="Aporia account disabled")
|
||||
async def test_llm_guard_triggered_safe_request():
|
||||
"""
|
||||
- Tests a request where no content mod is triggered
|
||||
- Assert that the guardrails applied are returned in the response headers
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
response, headers = await chat_completion(
|
||||
session,
|
||||
os.environ["LITELLM_MASTER_KEY"],
|
||||
model="fake-openai-endpoint",
|
||||
messages=[{"role": "user", "content": f"Hello what's the weather"}],
|
||||
guardrails=[
|
||||
"aporia-post-guard",
|
||||
"aporia-pre-guard",
|
||||
],
|
||||
)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
print("response=", response, "response headers", headers)
|
||||
|
||||
assert "x-litellm-applied-guardrails" in headers
|
||||
|
||||
assert (
|
||||
headers["x-litellm-applied-guardrails"]
|
||||
== "aporia-pre-guard,aporia-post-guard"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip(reason="Aporia account disabled")
|
||||
async def test_llm_guard_triggered():
|
||||
"""
|
||||
- Tests a request where no content mod is triggered
|
||||
- Assert that the guardrails applied are returned in the response headers
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
with pytest.raises(Exception, match="Aporia detected and blocked PII") as exc_info:
|
||||
response, headers = await chat_completion(
|
||||
session,
|
||||
os.environ["LITELLM_MASTER_KEY"],
|
||||
model="fake-openai-endpoint",
|
||||
messages=[
|
||||
{"role": "user", "content": f"Hello my name is ishaan@berri.ai"}
|
||||
],
|
||||
guardrails=[
|
||||
"aporia-post-guard",
|
||||
"aporia-pre-guard",
|
||||
],
|
||||
)
|
||||
e = exc_info.value
|
||||
print(e)
|
||||
assert "Aporia detected and blocked PII" in str(e)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_llm_guard_triggered():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
import os
|
||||
import pytest
|
||||
import asyncio
|
||||
import aiohttp, openai
|
||||
from openai import OpenAI, AsyncOpenAI
|
||||
from typing import Optional, List, Union
|
||||
from litellm._uuid import uuid
|
||||
|
||||
|
||||
async def make_rerank_curl_request(
|
||||
session,
|
||||
key,
|
||||
query,
|
||||
documents,
|
||||
model="rerank-english-v3.0",
|
||||
top_n=3,
|
||||
):
|
||||
url = "http://0.0.0.0:4000/rerank"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
data = {
|
||||
"model": model,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"top_n": top_n,
|
||||
}
|
||||
|
||||
async with session.post(url, headers=headers, json=data) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(response_text)
|
||||
|
||||
return await response.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_rerank_on_proxy():
|
||||
"""
|
||||
Test litellm.rerank() on proxy
|
||||
|
||||
This SHOULD NOT call the pass through endpoints :)
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
docs = [
|
||||
"Carson City is the capital city of the American state of Nevada.",
|
||||
"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
|
||||
"Washington, D.C. is the capital of the United States.",
|
||||
"Capital punishment has existed in the United States since before it was a country.",
|
||||
]
|
||||
|
||||
try:
|
||||
response = await make_rerank_curl_request(
|
||||
session,
|
||||
os.environ["LITELLM_MASTER_KEY"],
|
||||
query="What is the capital of the United States?",
|
||||
documents=docs,
|
||||
)
|
||||
print("response=", response)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
pytest.fail("Rerank request failed")
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
"""
|
||||
This test ensures that the proxy can passthrough anthropic requests
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import anthropic
|
||||
import os
|
||||
|
||||
client = anthropic.Anthropic(
|
||||
base_url="http://0.0.0.0:4000/anthropic", api_key=os.environ["LITELLM_MASTER_KEY"]
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_basic_completion():
|
||||
print("making basic completion request to anthropic passthrough")
|
||||
response = client.messages.create(
|
||||
model="claude-sonnet-4-5-20250929",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "Say 'hello test' and nothing else"}],
|
||||
)
|
||||
print(response)
|
||||
|
||||
|
||||
def test_anthropic_streaming():
|
||||
print("making streaming request to anthropic passthrough")
|
||||
collected_output = []
|
||||
|
||||
with client.messages.stream(
|
||||
max_tokens=10,
|
||||
messages=[
|
||||
{"role": "user", "content": "Say 'hello stream test' and nothing else"}
|
||||
],
|
||||
model="claude-sonnet-4-5-20250929",
|
||||
) as stream:
|
||||
for text in stream.text_stream:
|
||||
collected_output.append(text)
|
||||
|
||||
full_response = "".join(collected_output)
|
||||
print(full_response)
|
||||
|
|
@ -94,36 +94,6 @@ class BaseAnthropicMessagesTest:
|
|||
print(f"Non-streaming response: {json.dumps(response, indent=2, default=str)}")
|
||||
return response
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_base(self):
|
||||
"""Base test for streaming requests"""
|
||||
request_params = self.model_config
|
||||
# Set up test parameters
|
||||
messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}]
|
||||
|
||||
# Prepare call arguments
|
||||
call_args = {
|
||||
"messages": messages,
|
||||
"max_tokens": 100,
|
||||
"stream": True,
|
||||
"client": AsyncHTTPHandler(),
|
||||
}
|
||||
|
||||
# Add any additional config from subclass
|
||||
call_args.update(request_params)
|
||||
|
||||
# Call the handler
|
||||
response = await litellm.anthropic.messages.acreate(**call_args)
|
||||
|
||||
collected_chunks = []
|
||||
if isinstance(response, AsyncIterator):
|
||||
async for chunk in response:
|
||||
print("chunk=", chunk)
|
||||
collected_chunks.append(chunk)
|
||||
|
||||
print("collected_chunks=", collected_chunks)
|
||||
return collected_chunks
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_format_consistency(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ class TestAnthropicDirectAPI(BaseAnthropicMessagesTest):
|
|||
"""Tests for direct Anthropic API calls"""
|
||||
|
||||
test_non_streaming_base = None
|
||||
test_streaming_base = None
|
||||
|
||||
@property
|
||||
def model_config(self) -> Dict[str, Any]:
|
||||
|
|
@ -88,8 +87,6 @@ class TestAnthropicDirectAPI(BaseAnthropicMessagesTest):
|
|||
class TestAnthropicBedrockAPI(BaseAnthropicMessagesTest):
|
||||
"""Tests for Anthropic via Bedrock"""
|
||||
|
||||
test_streaming_base = None
|
||||
|
||||
@property
|
||||
def model_config(self) -> Dict[str, Any]:
|
||||
return {
|
||||
|
|
@ -107,8 +104,6 @@ class TestAnthropicBedrockAPI(BaseAnthropicMessagesTest):
|
|||
class TestAnthropicOpenAIAPI(BaseAnthropicMessagesTest):
|
||||
"""Tests for OpenAI via Anthropic messages interface"""
|
||||
|
||||
test_streaming_base = None
|
||||
|
||||
@property
|
||||
def model_config(self) -> Dict[str, Any]:
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -365,40 +365,6 @@ async def test_callback_observing_stamp_before_pre_header_increment_fails_leaves
|
|||
assert await router.get_model_group_usage("gpt-5-mini") == (None, None)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_track_deployment_metrics(model_list):
|
||||
"""Test if the 'track_deployment_metrics' function is working correctly"""
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
|
@ -416,18 +382,6 @@ def test_track_deployment_metrics(model_list):
|
|||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_pass_through_assistants_endpoint_factory(model_list):
|
||||
"""Test if the 'pass_through_assistants_endpoint_factory' function is working correctly"""
|
||||
router = Router(model_list=model_list)
|
||||
|
|
|
|||
|
|
@ -66,62 +66,6 @@ async def test_health():
|
|||
assert total_model_count > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_readiness():
|
||||
"""
|
||||
Check if 200
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
url = "http://0.0.0.0:4000/health/readiness"
|
||||
async with session.get(url) as response:
|
||||
status = response.status
|
||||
response_json = await response.json()
|
||||
|
||||
print(response_json)
|
||||
assert "status" in response_json
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_readiness_details():
|
||||
"""
|
||||
Check if authenticated readiness diagnostics expose version metadata.
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
url = "http://0.0.0.0:4000/health/readiness/details"
|
||||
headers = {"Authorization": "Bearer " + os.environ["LITELLM_MASTER_KEY"]}
|
||||
async with session.get(url, headers=headers) as response:
|
||||
status = response.status
|
||||
response_json = await response.json()
|
||||
|
||||
print(response_json)
|
||||
assert "status" in response_json
|
||||
assert "litellm_version" in response_json
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_liveliness():
|
||||
"""
|
||||
Check if 200
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
url = "http://0.0.0.0:4000/health/liveliness"
|
||||
async with session.get(url) as response:
|
||||
status = response.status
|
||||
response_text = await response.text()
|
||||
|
||||
print(response_text)
|
||||
print()
|
||||
|
||||
if status != 200:
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routes():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -232,20 +232,6 @@ async def delete_key(session, get_key, auth_key=os.environ["LITELLM_MASTER_KEY"]
|
|||
return await response.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_delete():
|
||||
"""
|
||||
Delete key
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
key_gen = await generate_key(session=session, i=0)
|
||||
key = key_gen["key"]
|
||||
await delete_key(
|
||||
session=session,
|
||||
get_key=key,
|
||||
)
|
||||
|
||||
|
||||
async def get_key_info(session, call_key, get_key=None):
|
||||
"""
|
||||
Make sure only models user has access to are returned
|
||||
|
|
@ -381,62 +367,6 @@ async def get_spend_logs(session, request_id):
|
|||
return await response.json()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Hanging on ci/cd")
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_info_spend_values():
|
||||
"""
|
||||
Test to ensure spend is correctly calculated
|
||||
- create key
|
||||
- make completion call
|
||||
- assert cost is expected value
|
||||
"""
|
||||
|
||||
async def retry_request(func, *args, _max_attempts=5, **kwargs):
|
||||
for attempt in range(_max_attempts):
|
||||
try:
|
||||
return await func(*args, **kwargs)
|
||||
except aiohttp.client_exceptions.ClientOSError as e:
|
||||
if attempt + 1 == _max_attempts:
|
||||
raise # re-raise the last ClientOSError if all attempts failed
|
||||
print(f"Attempt {attempt+1} failed, retrying...")
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
## Test Spend Update ##
|
||||
# completion
|
||||
key_gen = await generate_key(session=session, i=0)
|
||||
key = key_gen["key"]
|
||||
response = await chat_completion(session=session, key=key)
|
||||
await asyncio.sleep(5)
|
||||
spend_logs = await retry_request(
|
||||
get_spend_logs, session=session, request_id=response["id"]
|
||||
)
|
||||
print(f"spend_logs: {spend_logs}")
|
||||
completion_tokens = spend_logs[0]["completion_tokens"]
|
||||
prompt_tokens = spend_logs[0]["prompt_tokens"]
|
||||
print(f"prompt_tokens: {prompt_tokens}; completion_tokens: {completion_tokens}")
|
||||
|
||||
litellm.set_verbose = True
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model="gpt-35-turbo",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
custom_llm_provider="azure",
|
||||
)
|
||||
print("prompt_cost: ", prompt_cost, "completion_cost: ", completion_cost)
|
||||
response_cost = prompt_cost + completion_cost
|
||||
print(f"response_cost: {response_cost}")
|
||||
await asyncio.sleep(5) # allow db log to be updated
|
||||
key_info = await get_key_info(session=session, get_key=key, call_key=key)
|
||||
print(
|
||||
f"response_cost: {response_cost}; key_info spend: {key_info['info']['spend']}"
|
||||
)
|
||||
rounded_response_cost = round(response_cost, 8)
|
||||
rounded_key_info_spend = round(key_info["info"]["spend"], 8)
|
||||
assert (
|
||||
rounded_response_cost == rounded_key_info_spend
|
||||
), f"Expected cost= {rounded_response_cost} != Tracked Cost={rounded_key_info_spend}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.flaky(retries=6, delay=2)
|
||||
@pytest.mark.skip(
|
||||
|
|
@ -524,28 +454,6 @@ async def test_key_with_budgets():
|
|||
assert reset_at_init_value != reset_at_new_value
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_info_spend_values_sagemaker():
|
||||
"""
|
||||
Tests the sync streaming loop to ensure spend is correctly calculated.
|
||||
- create key
|
||||
- make completion call
|
||||
- assert cost is expected value
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
## streaming - sagemaker
|
||||
key_gen = await generate_key(session=session, i=0, models=[])
|
||||
new_key = key_gen["key"]
|
||||
prompt_tokens, completion_tokens = await chat_completion_streaming(
|
||||
session=session, key=new_key, model="sagemaker-completion-model"
|
||||
)
|
||||
await asyncio.sleep(5) # allow db log to be updated
|
||||
key_info = await get_key_info(
|
||||
session=session, get_key=new_key, call_key=new_key
|
||||
)
|
||||
rounded_key_info_spend = round(key_info["info"]["spend"], 8)
|
||||
assert rounded_key_info_spend > 0
|
||||
# assert rounded_response_cost == rounded_key_info_spend
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -101,24 +101,6 @@ async def get_spend_logs(session, request_id=None, api_key=None):
|
|||
return await response.json()
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Flaky in CI: /spend/logs?request_id=... returns 500 even after a 20s wait for the spend log to be written. Spend-log accuracy is covered by tests/unit/proxy/spend_tracking/ and the proxy_spend_accuracy_tests CircleCI job."
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_logs():
|
||||
"""
|
||||
- Create key
|
||||
- Make call (makes sure it's in spend logs)
|
||||
- Get request id from logs
|
||||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
key_gen = await generate_key(session=session)
|
||||
key = key_gen["key"]
|
||||
response = await chat_completion(session=session, key=key)
|
||||
await asyncio.sleep(20)
|
||||
await get_spend_logs(session=session, request_id=response["id"])
|
||||
|
||||
|
||||
async def generate_org(session: aiohttp.ClientSession) -> dict:
|
||||
"""
|
||||
Generate a new organization using the API.
|
||||
|
|
@ -236,59 +218,3 @@ async def test_get_predicted_spend_logs():
|
|||
|
||||
assert "response" in result
|
||||
assert len(result["response"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="High traffic load test, meant to be run locally")
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_logs_high_traffic():
|
||||
"""
|
||||
- Create key
|
||||
- Make 30 concurrent calls
|
||||
- Get all logs for that key
|
||||
- Wait 10s
|
||||
- Assert it's 30
|
||||
"""
|
||||
|
||||
async def retry_request(func, *args, _max_attempts=5, **kwargs):
|
||||
for attempt in range(_max_attempts):
|
||||
try:
|
||||
return await func(*args, **kwargs)
|
||||
except (
|
||||
aiohttp.client_exceptions.ClientOSError,
|
||||
aiohttp.client_exceptions.ServerDisconnectedError,
|
||||
) as e:
|
||||
if attempt + 1 == _max_attempts:
|
||||
raise # re-raise the last ClientOSError if all attempts failed
|
||||
print(f"Attempt {attempt+1} failed, retrying...")
|
||||
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=600)
|
||||
) as session:
|
||||
start = time.time()
|
||||
key_gen = await generate_key(session=session)
|
||||
key = key_gen["key"]
|
||||
n = 1000
|
||||
tasks = [
|
||||
retry_request(
|
||||
chat_completion_high_traffic,
|
||||
session=session,
|
||||
key=key,
|
||||
model="azure-gpt-3.5",
|
||||
)
|
||||
for _ in range(n)
|
||||
]
|
||||
chat_completions = await asyncio.gather(*tasks)
|
||||
successful_completions = [c for c in chat_completions if c is not None]
|
||||
print(f"Num successful completions: {len(successful_completions)}")
|
||||
await asyncio.sleep(10)
|
||||
try:
|
||||
response = await retry_request(get_spend_logs, session=session, api_key=key)
|
||||
print(f"response: {response}")
|
||||
print(f"len responses: {len(response)}")
|
||||
assert len(response) == n
|
||||
print(n, time.time() - start, len(response))
|
||||
except Exception:
|
||||
print(n, time.time() - start, 0)
|
||||
raise Exception("it worked!")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -137,122 +137,6 @@ def test_add_single_member(api_client, new_team):
|
|||
), f"Team size did not increase by 1 (was {initial_size}, now {updated_size})"
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Flaky in CI: /team/info?team_id=... intermittently returns 404/400 mid-loop after add_team_member calls. Single-member coverage in test_add_single_member is sufficient; team-member CRUD is also covered by tests/unit/proxy/management_endpoints/."
|
||||
)
|
||||
def test_add_multiple_members(api_client, new_team):
|
||||
"""Test adding multiple members to a new team"""
|
||||
# Get initial team size
|
||||
initial_info = api_client.get_team_info(new_team)
|
||||
initial_size = len(initial_info["team_info"]["members_with_roles"])
|
||||
|
||||
# Add 10 members
|
||||
added_emails = []
|
||||
for i in range(10):
|
||||
email = f"pytest_user_{uuid.uuid4().hex[:6]}@mycompany.com"
|
||||
added_emails.append(email)
|
||||
|
||||
logger.info(f"Adding member {i+1}/10: {email}")
|
||||
api_client.add_team_member(new_team, email, "user")
|
||||
|
||||
# Allow time for system to process
|
||||
time.sleep(1)
|
||||
|
||||
# Verify after each addition
|
||||
current_info = api_client.get_team_info(new_team)
|
||||
current_size = len(current_info["team_info"]["members_with_roles"])
|
||||
|
||||
# Assertions for each addition
|
||||
assert verify_member_in_team(
|
||||
current_info, email
|
||||
), f"Member {email} not found in team"
|
||||
assert (
|
||||
current_size == initial_size + i + 1
|
||||
), f"Team size incorrect after adding {email}"
|
||||
|
||||
# Final verification
|
||||
final_info = api_client.get_team_info(new_team)
|
||||
final_size = len(final_info["team_info"]["members_with_roles"])
|
||||
|
||||
# Final assertions
|
||||
assert (
|
||||
final_size == initial_size + 10
|
||||
), f"Final team size incorrect (expected {initial_size + 10}, got {final_size})"
|
||||
for email in added_emails:
|
||||
assert verify_member_in_team(
|
||||
final_info, email
|
||||
), f"Member {email} not found in final team check"
|
||||
|
||||
|
||||
def test_team_info_structure(api_client, new_team):
|
||||
"""Test the structure of team info response"""
|
||||
team_info = api_client.get_team_info(new_team)
|
||||
|
||||
# Verify required fields exist
|
||||
assert "team_id" in team_info
|
||||
assert "team_info" in team_info
|
||||
assert "members_with_roles" in team_info["team_info"]
|
||||
assert "models" in team_info["team_info"]
|
||||
|
||||
# Verify member structure
|
||||
if team_info["team_info"]["members_with_roles"]:
|
||||
member = team_info["team_info"]["members_with_roles"][0]
|
||||
assert "user_id" in member
|
||||
assert "role" in member
|
||||
|
||||
|
||||
def test_error_handling(api_client):
|
||||
"""Test error handling for invalid team ID"""
|
||||
with pytest.raises(requests.exceptions.HTTPError):
|
||||
api_client.get_team_info("invalid-team-id")
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Flaky in CI: /team/info?team_id=... intermittently returns 404 after add_team_member calls, same race documented for test_add_multiple_members. Duplicate-prevention is covered by test_update_team_members_list_duplicate_prevention in tests/unit/proxy/management_endpoints/test_team_endpoints.py."
|
||||
)
|
||||
def test_duplicate_user_addition(api_client, new_team):
|
||||
"""Test that adding the same user twice is handled appropriately"""
|
||||
# Add user first time
|
||||
test_email = f"pytest_user_{uuid.uuid4().hex[:6]}@mycompany.com"
|
||||
initial_response = api_client.add_team_member(new_team, test_email, "user")
|
||||
|
||||
# Allow time for system to process
|
||||
time.sleep(1)
|
||||
|
||||
# Get team info after first addition
|
||||
team_info_after_first = api_client.get_team_info(new_team)
|
||||
size_after_first = len(team_info_after_first["team_info"]["members_with_roles"])
|
||||
|
||||
logger.info(f"First addition completed. Team size: {size_after_first}")
|
||||
|
||||
# Attempt to add same user again
|
||||
with pytest.raises(requests.exceptions.HTTPError):
|
||||
api_client.add_team_member(new_team, test_email, "user")
|
||||
|
||||
# Allow time for system to process
|
||||
time.sleep(1)
|
||||
|
||||
# Get team info after second addition attempt
|
||||
team_info_after_second = api_client.get_team_info(new_team)
|
||||
size_after_second = len(team_info_after_second["team_info"]["members_with_roles"])
|
||||
|
||||
# Verify team size didn't change
|
||||
assert (
|
||||
size_after_second == size_after_first
|
||||
), f"Team size changed after duplicate addition (was {size_after_first}, now {size_after_second})"
|
||||
|
||||
# Verify user appears exactly once
|
||||
user_count = sum(
|
||||
1
|
||||
for member in team_info_after_second["team_info"]["members_with_roles"]
|
||||
if member["user_id"] == test_email
|
||||
)
|
||||
assert user_count == 1, f"User appears {user_count} times in team (expected 1)"
|
||||
|
||||
logger.info(f"Duplicate addition attempted. Final team size: {size_after_second}")
|
||||
logger.info(f"Number of times user appears in team: {user_count}")
|
||||
|
||||
|
||||
def test_member_deletion(api_client, new_team):
|
||||
"""Test that member deletion works correctly and removes all instances of a user"""
|
||||
# Add a test user
|
||||
|
|
|
|||
|
|
@ -110,16 +110,6 @@ async def test_user_info():
|
|||
assert status == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_update():
|
||||
"""
|
||||
Create user
|
||||
Update user access to new model
|
||||
Make chat completion call
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Frequent check on ci/cd leads to read timeout issue.")
|
||||
@pytest.mark.asyncio
|
||||
async def test_users_budgets_reset():
|
||||
|
|
@ -182,38 +172,6 @@ async def chat_completion_streaming(session, key, model="gpt-4"):
|
|||
continue
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Global proxy now tracked via `/global/spend/logs`")
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_proxy_budget_update():
|
||||
"""
|
||||
- Get proxy current spend
|
||||
- Make chat completion call (normal)
|
||||
- Assert spend increased
|
||||
- Make chat completion call (streaming)
|
||||
- Assert spend increased
|
||||
"""
|
||||
get_user = f"litellm-proxy-budget"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
user_info = await get_user_info(
|
||||
session=session, get_user=get_user, call_user=os.environ["LITELLM_MASTER_KEY"]
|
||||
)
|
||||
original_spend = user_info["user_info"]["spend"]
|
||||
await chat_completion(session=session, key=os.environ["LITELLM_MASTER_KEY"])
|
||||
await asyncio.sleep(5) # let db update
|
||||
user_info = await get_user_info(
|
||||
session=session, get_user=get_user, call_user=os.environ["LITELLM_MASTER_KEY"]
|
||||
)
|
||||
new_spend = user_info["user_info"]["spend"]
|
||||
print(f"new_spend: {new_spend}; original_spend: {original_spend}")
|
||||
assert new_spend > original_spend
|
||||
await chat_completion_streaming(session=session, key=os.environ["LITELLM_MASTER_KEY"])
|
||||
await asyncio.sleep(5) # let db update
|
||||
user_info = await get_user_info(
|
||||
session=session, get_user=get_user, call_user=os.environ["LITELLM_MASTER_KEY"]
|
||||
)
|
||||
new_new_spend = user_info["user_info"]["spend"]
|
||||
print(f"new_spend: {new_spend}; original_spend: {original_spend}")
|
||||
assert new_new_spend > new_spend
|
||||
|
||||
|
||||
import json
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue