scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

Merge pull request #41527 from BerriAI/litellm_/monitor-cci-failures-ac97a9

Browse source 
test: fix seven tests left stale by #41311, #41337, #39996, #41310, #41289 and #41315
This commit is contained in:
yuneng-jiang 2026-09-16 21:25:51 -07:00 • committed by GitHub
commit 5ef40a630b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 81 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts
View file

@ -47,9 +47,11 @@ test.describe("Auto Router template select anchoring", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("opens the options below the trigger when there is room below it", async ({ page }) => {
await page.setViewportSize({ width: 1280, height: 900 });
const viewport = { width: 1280, height: 900 };
await page.setViewportSize(viewport);
const trigger = await openTemplateSelect(page);
await trigger.scrollIntoViewIfNeeded();
await trigger.evaluate((element) => element.scrollIntoView({ block: "start" }));
await expect.poll(async () => (await trigger.boundingBox())?.y).toBeLessThan(viewport.height / 2);
await trigger.click();
await expect(page.getByRole("listbox")).toBeVisible();

6
tests/integration/_support/client.py
View file

@ -139,8 +139,10 @@ class Scenario:
def delete_key(self, token: str) -> None:
self.gateway.post("/key/delete", {"keys": [token]})
response: Final = self.gateway.request("GET", "/key/info", params={"key": sha256(token.encode()).hexdigest()})
assert response.status_code == 404, f"Deleted key remains readable: {response.status_code}"
hashed: Final = sha256(token.encode()).hexdigest()
assert read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed,)) == []
info: Final = object_value(self.gateway.get("/key/info", {"key": hashed})["info"])
assert info["status"] == "deleted", f"Deleted key still served as live: {info['status']}"
def delete_model(self, identity: str) -> None:
self.gateway.post("/model/delete", {"id": identity})

45
tests/integration/authorization/test_warmed_policy.py
View file

@ -1,10 +1,12 @@
from contextlib import ExitStack
from collections.abc import Iterator
from contextlib import ExitStack, contextmanager
from hashlib import sha256
from typing import Final
import os
import psycopg
import pytest
from pydantic import JsonValue
from hypothesis import strategies as st
from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test
@ -134,37 +136,58 @@ def test_scim_deactivation_blocks_null_and_false_keys_but_preserves_other_owners
assert_serving(gateway, model, token, 200)
def _set_team_admin_editable_fields(gateway: Gateway, fields: list[JsonValue]) -> None:
response: Final = gateway.request("PATCH", "/update/ui_settings", {"team_admin_editable_team_fields": fields})
assert response.status_code == 200, response.text
@contextmanager
def _team_admins_may_edit(gateway: Gateway, fields: list[JsonValue]) -> Iterator[None]:
original: Final = object_value(gateway.get("/get/ui_settings")["values"]).get("team_admin_editable_team_fields")
_set_team_admin_editable_fields(gateway, fields)
try:
yield
finally:
_set_team_admin_editable_fields(gateway, original if isinstance(original, list) else [])
@pytest.mark.covers("mgmt.team.member_update.demoted_role_cannot_write")
def test_warmed_team_role_demotion_prevents_later_management_writes(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
with gateway.scenario() as scenario, _team_admins_may_edit(gateway, ["tpm_limit"]):
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
team: Final = scenario.team(models=[model], members_with_roles=[{"user_id": user, "role": "admin"}])
control_team: Final = scenario.team(models=[model])
team: Final = scenario.team(
models=[model], tpm_limit=1000, members_with_roles=[{"user_id": user, "role": "admin"}]
)
control_team: Final = scenario.team(models=[model], tpm_limit=1000)
caller: Final = scenario.key(
user_id=user, team_id=team, models=[model], allowed_routes=["/team/update", "/v1/chat/completions"]
)
gateway.chat(model, key=caller)
changed: Final = gateway.request("POST", "/team/update", {"team_id": team, "team_alias": "permitted"}, key=caller)
changed: Final = gateway.request("POST", "/team/update", {"team_id": team, "tpm_limit": 5000}, key=caller)
assert changed.status_code == 200, changed.text
assert read_rows('SELECT tpm_limit FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) == [
{"tpm_limit": 5000}
]
unrelated_before: Final = read_rows(
'SELECT team_alias FROM "LiteLLM_TeamTable" WHERE team_id = %s', (control_team,)
'SELECT tpm_limit FROM "LiteLLM_TeamTable" WHERE team_id = %s', (control_team,)
)
unrelated: Final = gateway.request(
"POST", "/team/update", {"team_id": control_team, "team_alias": "must-not-persist"}, key=caller
"POST", "/team/update", {"team_id": control_team, "tpm_limit": 7000}, key=caller
)
assert unrelated.status_code == 403, unrelated.text
assert read_rows(
'SELECT team_alias FROM "LiteLLM_TeamTable" WHERE team_id = %s', (control_team,)
'SELECT tpm_limit FROM "LiteLLM_TeamTable" WHERE team_id = %s', (control_team,)
) == unrelated_before
gateway.post("/team/member_update", {"team_id": team, "user_id": user, "role": "user"})
for target in (team, control_team):
before: Final = read_rows('SELECT team_alias FROM "LiteLLM_TeamTable" WHERE team_id = %s', (target,))
before: Final = read_rows('SELECT tpm_limit FROM "LiteLLM_TeamTable" WHERE team_id = %s', (target,))
denied: Final = gateway.request(
"POST", "/team/update", {"team_id": target, "team_alias": "must-not-persist"}, key=caller
"POST", "/team/update", {"team_id": target, "tpm_limit": 9000}, key=caller
)
assert denied.status_code == 403, denied.text
assert read_rows('SELECT team_alias FROM "LiteLLM_TeamTable" WHERE team_id = %s', (target,)) == before
after: Final = read_rows('SELECT tpm_limit FROM "LiteLLM_TeamTable" WHERE team_id = %s', (target,))
assert after == before
roster: Final = read_rows('SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,))
members: Final = roster[0]["members_with_roles"]
assert isinstance(members, list)

7
tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py
View file

@ -26,6 +26,7 @@ from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfi
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseCompletedEvent,
ResponseFailedEvent,
ResponseIncompleteEvent,
@ -69,6 +70,7 @@ class TestBaseResponsesAPIStreamingIterator:
mock_responses_api_response = Mock(spec=ResponsesAPIResponse)
mock_responses_api_response.id = "resp_u2028"
mock_responses_api_response.usage = ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5)
mock_completed_event = Mock(spec=ResponseCompletedEvent)
mock_completed_event.type = ResponsesAPIStreamEvents.RESPONSE_COMPLETED
mock_completed_event.response = mock_responses_api_response
@ -123,6 +125,7 @@ class TestBaseResponsesAPIStreamingIterator:
# Mock the _update_responses_api_response_id_with_model_id method
updated_response = Mock(spec=ResponsesAPIResponse)
updated_response.id = "updated_response_id"
updated_response.usage = ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5)
# Create the iterator instance
iterator = BaseResponsesAPIStreamingIterator(
@ -524,7 +527,7 @@ class TestBaseResponsesAPIStreamingIterator:
"type": "server_error",
"message": "The model encountered an error",
}
mock_responses_api_response.usage = None
mock_responses_api_response.usage = ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5)
mock_failed_event = Mock(spec=ResponseFailedEvent)
mock_failed_event.type = ResponsesAPIStreamEvents.RESPONSE_FAILED
@ -604,7 +607,7 @@ class TestBaseResponsesAPIStreamingIterator:
mock_responses_api_response = Mock(spec=ResponsesAPIResponse)
mock_responses_api_response.id = "resp_incomplete_123"
mock_responses_api_response.incomplete_details = {"reason": "max_output_tokens"}
mock_responses_api_response.usage = None
mock_responses_api_response.usage = ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5)
mock_incomplete_event = Mock(spec=ResponseIncompleteEvent)
mock_incomplete_event.type = ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE

45
tests/local_testing/test_router_cooldown_handlers.py
View file

@ -833,45 +833,38 @@ def test_router_fallbacks_with_cooldowns_and_model_id():
@pytest.mark.asyncio()
async def test_router_fallbacks_with_cooldowns_and_dynamic_credentials():
"""
Ensure cooldown on credential 1 does not affect credential 2
A 429 answered to a caller-supplied credential cools down none of the shared deployments,
so the next credential still reaches them, while a 429 owned by a shared deployment does
"""
from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments
litellm._turn_on_debug()
router = Router(
model_list=[
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "gpt-3.5-turbo", "rpm": 1},
"model_info": {
"id": "123",
},
"litellm_params": {"model": "gpt-3.5-turbo"},
"model_info": {"id": deployment_id},
}
]
for deployment_id in ("123", "456")
],
num_retries=0,
)
messages = [{"role": "user", "content": "hi"}]
## trigger ratelimit
try:
with pytest.raises(litellm.RateLimitError):
await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "hi"}],
api_key="my-bad-key-1",
mock_response="litellm.RateLimitError",
model="gpt-3.5-turbo", messages=messages, api_key="my-bad-key-1", mock_response="litellm.RateLimitError"
)
pytest.fail("Expected RateLimitError")
except litellm.RateLimitError:
pass
await asyncio.sleep(1)
assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
cooldown_list = await _async_get_cooldown_deployments(
litellm_router_instance=router, parent_otel_span=None
response = await router.acompletion(
model="gpt-3.5-turbo", messages=messages, api_key="my-good-key-2", mock_response="served with credential 2"
)
print("cooldown_list: ", cooldown_list)
assert len(cooldown_list) == 1
assert response.choices[0].message.content == "served with credential 2"
await router.acompletion(
model="gpt-3.5-turbo",
api_key=os.getenv("OPENAI_API_KEY"),
messages=[{"role": "user", "content": "hi"}],
)
with pytest.raises(litellm.RateLimitError):
await router.acompletion(model="gpt-3.5-turbo", messages=messages, mock_response="litellm.RateLimitError")
await asyncio.sleep(1)
cooled_down = await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None)
assert len(cooled_down) == 1 and cooled_down[0] in {"123", "456"}

29
tests/local_testing/test_text_completion.py
View file

@ -12,7 +12,6 @@ from unittest.mock import MagicMock, patch
import pytest
import litellm
from tests._live_test_helpers import cheapest_together_chat_model
from litellm import (
RateLimitError,
TextCompletionResponse,
@ -4023,27 +4022,27 @@ def test_async_text_completion():
asyncio.run(test_get_response())
@pytest.mark.flaky(retries=6, delay=1)
def test_async_text_completion_together_ai():
litellm.set_verbose = True
print("test_async_text_completion")
from openai import AsyncOpenAI
async def test_get_response():
try:
client = AsyncOpenAI(api_key="my-fake-key")
async def run_call():
with patch.object(client.completions.with_raw_response, "create", side_effect=mock_post) as mock_call:
response = await litellm.atext_completion(
model=cheapest_together_chat_model(),
model="together_ai/Qwen/Qwen2-1.5B-Instruct",
prompt="good morning",
max_tokens=10,
client=client,
)
print(f"response: {response}")
except litellm.RateLimitError as e:
print(e)
except litellm.Timeout as e:
print(e)
except Exception as e:
pytest.fail("An unexpected error occurred")
return response, mock_call.call_args.kwargs
asyncio.run(test_get_response())
response, sent = asyncio.run(run_call())
assert sent["model"] == "Qwen/Qwen2-1.5B-Instruct"
assert sent["prompt"] == "good morning"
assert sent["max_tokens"] == 10
assert response.choices[0].text == ") might be faster than then answering, and the added time it takes for the"
assert response.usage.total_tokens == 18
# test_async_text_completion()

2
tests/test_openai_endpoints.py
View file

@ -307,7 +307,7 @@ async def test_chat_completion():
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}],
)
assert "is not available for this API key" in str(e)
assert "is not available for this API key" in str(e.value)
@pytest.mark.asyncio