scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

test: repair stale CircleCI contracts

Browse source
This commit is contained in:
Yuneng Jiang 2026-08-08 12:19:22 -07:00
parent 8b16ee1dc2
commit 0d7f7c689a
No known key found for this signature in database
8 changed files with 81 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
litellm/proxy/db/autorouter_session_rollup.py
View file

@ -33,6 +33,53 @@ if TYPE_CHECKING:
CACHE_TTL_5M_SECONDS: Final = 300
CACHE_TTL_1H_SECONDS: Final = 3600
AUTOROUTER_BENCHMARKS_SQL: Final = """
WITH windowed AS (
SELECT * FROM "LiteLLM_AutoRouterSession"
WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp
),
tier_maps AS (
SELECT router_name, router_type, jsonb_object_agg(tier, tier_turns) AS tier_turns
FROM (
SELECT router_name, router_type, kv.key AS tier, SUM((kv.value)::int)::int AS tier_turns
FROM windowed, LATERAL jsonb_each_text(tier_turns) AS kv
GROUP BY router_name, router_type, kv.key
) per_tier
GROUP BY router_name, router_type
)
SELECT
agg.*,
COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns
FROM (
SELECT
router_name,
router_type,
COUNT(*)::int AS sessions,
COALESCE(SUM(turns), 0)::int AS turns,
COALESCE(SUM(unordered_turns), 0)::int AS unordered_turns,
COALESCE(SUM(covered_turns), 0)::int AS covered_turns,
COALESCE(SUM(cache_hits), 0)::int AS cache_hits,
COALESCE(SUM(same_model_turns), 0)::int AS same_model_turns,
COALESCE(SUM(same_model_hits), 0)::int AS same_model_hits,
COALESCE(SUM(first_visit_turns), 0)::int AS first_visit_turns,
COALESCE(SUM(first_visit_hits), 0)::int AS first_visit_hits,
COALESCE(SUM(return_turns), 0)::int AS return_turns,
COALESCE(SUM(return_hits), 0)::int AS return_hits,
COALESCE(SUM(return_expired_misses), 0)::int AS return_expired_misses,
COALESCE(SUM(return_within_ttl_misses), 0)::int AS return_within_ttl_misses,
COALESCE(SUM(ttl_5m_turns), 0)::int AS ttl_5m_turns,
COALESCE(SUM(ttl_1h_turns), 0)::int AS ttl_1h_turns,
COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens,
COALESCE(SUM(spend), 0)::float8 AS spend,
COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds
FROM windowed
GROUP BY router_name, router_type
) agg
LEFT JOIN tier_maps USING (router_name, router_type)
ORDER BY agg.spend DESC
"""
@dataclass(frozen=True, slots=True)
class AutoRouterTurnTransaction:

50
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -26,6 +26,7 @@ from litellm.proxy.auth.auth_checks import (
can_key_call_resolved_model,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.db.autorouter_session_rollup import AUTOROUTER_BENCHMARKS_SQL
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.repositories.team_repository import TeamRepository
from litellm.router_strategy.complexity_router import ComplexityRouter
@ -285,53 +286,6 @@ class _SessionAggRow(BaseModel):
_SESSION_AGG_ROWS: Final = TypeAdapter(list[_SessionAggRow])
_BENCHMARKS_SQL: Final = """
WITH windowed AS (
SELECT * FROM "LiteLLM_AutoRouterSession"
WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp
),
tier_maps AS (
SELECT router_name, router_type, jsonb_object_agg(tier, tier_turns) AS tier_turns
FROM (
SELECT router_name, router_type, kv.key AS tier, SUM((kv.value)::int)::int AS tier_turns
FROM windowed, LATERAL jsonb_each_text(tier_turns) AS kv
GROUP BY router_name, router_type, kv.key
) per_tier
GROUP BY router_name, router_type
)
SELECT
agg.*,
COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns
FROM (
SELECT
router_name,
router_type,
COUNT(*)::int AS sessions,
COALESCE(SUM(turns), 0)::int AS turns,
COALESCE(SUM(unordered_turns), 0)::int AS unordered_turns,
COALESCE(SUM(covered_turns), 0)::int AS covered_turns,
COALESCE(SUM(cache_hits), 0)::int AS cache_hits,
COALESCE(SUM(same_model_turns), 0)::int AS same_model_turns,
COALESCE(SUM(same_model_hits), 0)::int AS same_model_hits,
COALESCE(SUM(first_visit_turns), 0)::int AS first_visit_turns,
COALESCE(SUM(first_visit_hits), 0)::int AS first_visit_hits,
COALESCE(SUM(return_turns), 0)::int AS return_turns,
COALESCE(SUM(return_hits), 0)::int AS return_hits,
COALESCE(SUM(return_expired_misses), 0)::int AS return_expired_misses,
COALESCE(SUM(return_within_ttl_misses), 0)::int AS return_within_ttl_misses,
COALESCE(SUM(ttl_5m_turns), 0)::int AS ttl_5m_turns,
COALESCE(SUM(ttl_1h_turns), 0)::int AS ttl_1h_turns,
COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens,
COALESCE(SUM(spend), 0)::float8 AS spend,
COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds
FROM windowed
GROUP BY router_name, router_type
) agg
LEFT JOIN tier_maps USING (router_name, router_type)
ORDER BY agg.spend DESC
"""
def _parse_benchmark_day(value: str) -> datetime:
try:
@ -455,7 +409,7 @@ async def get_auto_router_benchmarks(
raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date")
raw_rows: Final = await prisma_client.db.query_raw(
_BENCHMARKS_SQL,
AUTOROUTER_BENCHMARKS_SQL,
start_day.isoformat(),
(end_day + timedelta(days=1)).isoformat(),
)

2
tests/agent_tests/test_a2a_agent.py
View file

@ -40,7 +40,7 @@ class MockA2AClient:
name="mock-agent", url="http://mock-agent.local"
)
async def send_message(self, request):
async def send_message(self, request, *, context=None):
from a2a.compat.v0_3.conversions import pb2_v10
for text in ("hel", "hello"):

24
tests/litellm_utils_tests/test_proxy_budget_reset.py
View file

@ -622,8 +622,12 @@ async def test_service_logger_keys_success():
logger success hook is called with the correct event metadata and no exception is logged.
"""
keys = [
{"id": "key1", "spend": 10.0, "budget_duration": 60, "token": "key1"},
{"id": "key2", "spend": 15.0, "budget_duration": 60, "token": "key2"},
_attrify(
{"id": "key1", "spend": 10.0, "budget_duration": 60, "token": "key1"}
),
_attrify(
{"id": "key2", "spend": 15.0, "budget_duration": 60, "token": "key2"}
),
]
prisma_client = MagicMock()
prisma_client.get_data = AsyncMock(return_value=keys)
@ -740,8 +744,12 @@ async def test_service_logger_users_success():
the correct metadata and no exception is logged.
"""
users = [
{"id": "user1", "spend": 20.0, "budget_duration": 120, "user_id": "user1"},
{"id": "user2", "spend": 25.0, "budget_duration": 120, "user_id": "user2"},
_attrify(
{"id": "user1", "spend": 20.0, "budget_duration": 120, "user_id": "user1"}
),
_attrify(
{"id": "user2", "spend": 25.0, "budget_duration": 120, "user_id": "user2"}
),
]
prisma_client = MagicMock()
prisma_client.get_data = AsyncMock(return_value=users)
@ -853,8 +861,12 @@ async def test_service_logger_teams_success():
the proper metadata and nothing is logged as an exception.
"""
teams = [
{"id": "team1", "spend": 30.0, "budget_duration": 180, "team_id": "team1"},
{"id": "team2", "spend": 35.0, "budget_duration": 180, "team_id": "team2"},
_attrify(
{"id": "team1", "spend": 30.0, "budget_duration": 180, "team_id": "team1"}
),
_attrify(
{"id": "team2", "spend": 35.0, "budget_duration": 180, "team_id": "team2"}
),
]
prisma_client = MagicMock()
prisma_client.get_data = AsyncMock(return_value=teams)

4
tests/llm_responses_api_testing/base_responses_api.py
View file

@ -338,7 +338,7 @@ class BaseResponsesAPITest(ABC):
)
assert result is not None
assert result.id == response.id
assert result.output == response.output
assert result.output_text == response.output_text
else:
raise ValueError("response is not a ResponsesAPIResponse")
else:
@ -352,7 +352,7 @@ class BaseResponsesAPITest(ABC):
)
assert result is not None
assert result.id == response.id
assert result.output == response.output
assert result.output_text == response.output_text
else:
raise ValueError("response is not a ResponsesAPIResponse")

16
tests/proxy_behavior/spend/test_autorouter_session_rollup.py
View file

@ -12,8 +12,10 @@ from typing import Final
import pytest
from litellm.proxy.db.autorouter_session_rollup import UPSERT_AUTOROUTER_SESSION_SQL
from litellm.proxy.management_endpoints.auto_router_endpoints import _BENCHMARKS_SQL
from litellm.proxy.db.autorouter_session_rollup import (
AUTOROUTER_BENCHMARKS_SQL,
UPSERT_AUTOROUTER_SESSION_SQL,
)
pytestmark = pytest.mark.asyncio(loop_scope="session")
@ -164,7 +166,7 @@ async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db):
await _turn(db, key, "A", T0 - timedelta(days=40), session_id=out_of_window, router=router)
rows = await db.query_raw(
_BENCHMARKS_SQL,
AUTOROUTER_BENCHMARKS_SQL,
(T0 - timedelta(days=1)).isoformat(),
(T0 + timedelta(days=1)).isoformat(),
)
@ -186,7 +188,7 @@ async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db
await _turn(db, key, "A", T0 + timedelta(seconds=10), session_id=f"s-{uuid.uuid4()}", router=router, router_type="quality")
rows = await db.query_raw(
_BENCHMARKS_SQL,
AUTOROUTER_BENCHMARKS_SQL,
(T0 - timedelta(days=1)).isoformat(),
(T0 + timedelta(days=1)).isoformat(),
)
@ -248,7 +250,7 @@ async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db):
await _turn(db, key, "C", T0 + timedelta(seconds=30), session_id=f"s-{uuid.uuid4()}", router=router, tier=None)
rows = await db.query_raw(
_BENCHMARKS_SQL,
AUTOROUTER_BENCHMARKS_SQL,
(T0 - timedelta(days=1)).isoformat(),
(T0 + timedelta(days=1)).isoformat(),
)
@ -275,7 +277,7 @@ async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(d
)
rows = await db.query_raw(
_BENCHMARKS_SQL,
AUTOROUTER_BENCHMARKS_SQL,
(T0 - timedelta(days=1)).isoformat(),
(T0 + timedelta(days=1)).isoformat(),
)
@ -289,7 +291,7 @@ async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db):
await _turn(db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, tier=None)
rows = await db.query_raw(
_BENCHMARKS_SQL,
AUTOROUTER_BENCHMARKS_SQL,
(T0 - timedelta(days=1)).isoformat(),
(T0 + timedelta(days=1)).isoformat(),
)

23
ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
View file

@ -919,7 +919,7 @@ describe("TeamInfoView", () => {
});
};
it("prefills pairs from team metadata, hides UI-managed keys, and round-trips typed values on save", async () => {
it("should preserve metadata types and hide managed keys", async () => {
const user = userEvent.setup({ delay: null });
vi.mocked(networking.teamInfoCall).mockResolvedValue(
createMockTeamData({
@ -964,27 +964,6 @@ describe("TeamInfoView", () => {
expect(updateArg.model_tpm_limit).toEqual({ "gpt-4": 100 });
});
it("includes a newly added pair in the team update", async () => {
const user = userEvent.setup({ delay: null });
vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] }));
vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any);
renderWithProviders(<TeamInfoView {...defaultProps} />);
await openSettingsEditor(user);
await user.click(screen.getByRole("button", { name: /add key-value pair/i }));
await user.type(screen.getByPlaceholderText("Key"), "cost_center");
await user.type(screen.getByPlaceholderText("Value"), "eng-1");
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() => {
expect(networking.teamUpdateCall).toHaveBeenCalled();
});
expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1].metadata).toMatchObject({ cost_center: "eng-1" });
});
it("should keep declared keys as ordinary prefilled rows and submit the edited value", async () => {
const user = userEvent.setup({ delay: null });
vi.mocked(useTeamMetadataSchema).mockReturnValue({

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -24313,7 +24313,7 @@ export interface components {
* @description Default role assigned to new users created
* @default internal_user_viewer
*/
user_role: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null;
user_role: ("internal_user" | "internal_user_viewer" | "proxy_admin" | "proxy_admin_viewer") | null;
};
/**
* DefaultTeamSSOParams