From 0d7f7c689a58aeea6cd26b8cb9fba90b31be8955 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 8 Aug 2026 12:19:22 -0700 Subject: [PATCH] test: repair stale CircleCI contracts --- litellm/proxy/db/autorouter_session_rollup.py | 47 +++++++++++++++++ .../auto_router_endpoints.py | 50 +------------------ tests/agent_tests/test_a2a_agent.py | 2 +- .../test_proxy_budget_reset.py | 24 ++++++--- .../base_responses_api.py | 4 +- .../spend/test_autorouter_session_rollup.py | 16 +++--- .../src/components/team/TeamInfo.test.tsx | 23 +-------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 8 files changed, 81 insertions(+), 87 deletions(-) diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index b1f074c26b7..9732b1d7402 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -33,6 +33,53 @@ if TYPE_CHECKING: CACHE_TTL_5M_SECONDS: Final = 300 CACHE_TTL_1H_SECONDS: Final = 3600 +AUTOROUTER_BENCHMARKS_SQL: Final = """ +WITH windowed AS ( + SELECT * FROM "LiteLLM_AutoRouterSession" + WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp +), +tier_maps AS ( + SELECT router_name, router_type, jsonb_object_agg(tier, tier_turns) AS tier_turns + FROM ( + SELECT router_name, router_type, kv.key AS tier, SUM((kv.value)::int)::int AS tier_turns + FROM windowed, LATERAL jsonb_each_text(tier_turns) AS kv + GROUP BY router_name, router_type, kv.key + ) per_tier + GROUP BY router_name, router_type +) +SELECT + agg.*, + COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns +FROM ( +SELECT + router_name, + router_type, + COUNT(*)::int AS sessions, + COALESCE(SUM(turns), 0)::int AS turns, + COALESCE(SUM(unordered_turns), 0)::int AS unordered_turns, + COALESCE(SUM(covered_turns), 0)::int AS covered_turns, + COALESCE(SUM(cache_hits), 0)::int AS cache_hits, + COALESCE(SUM(same_model_turns), 0)::int AS same_model_turns, + COALESCE(SUM(same_model_hits), 0)::int AS same_model_hits, + COALESCE(SUM(first_visit_turns), 0)::int AS first_visit_turns, + COALESCE(SUM(first_visit_hits), 0)::int AS first_visit_hits, + COALESCE(SUM(return_turns), 0)::int AS return_turns, + COALESCE(SUM(return_hits), 0)::int AS return_hits, + COALESCE(SUM(return_expired_misses), 0)::int AS return_expired_misses, + COALESCE(SUM(return_within_ttl_misses), 0)::int AS return_within_ttl_misses, + COALESCE(SUM(ttl_5m_turns), 0)::int AS ttl_5m_turns, + COALESCE(SUM(ttl_1h_turns), 0)::int AS ttl_1h_turns, + COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens, + COALESCE(SUM(spend), 0)::float8 AS spend, + COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend, + COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds +FROM windowed +GROUP BY router_name, router_type +) agg +LEFT JOIN tier_maps USING (router_name, router_type) +ORDER BY agg.spend DESC +""" + @dataclass(frozen=True, slots=True) class AutoRouterTurnTransaction: diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d4221845b0c..8b6aafea751 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -26,6 +26,7 @@ from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.autorouter_session_rollup import AUTOROUTER_BENCHMARKS_SQL from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.repositories.team_repository import TeamRepository from litellm.router_strategy.complexity_router import ComplexityRouter @@ -285,53 +286,6 @@ class _SessionAggRow(BaseModel): _SESSION_AGG_ROWS: Final = TypeAdapter(list[_SessionAggRow]) -_BENCHMARKS_SQL: Final = """ -WITH windowed AS ( - SELECT * FROM "LiteLLM_AutoRouterSession" - WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp -), -tier_maps AS ( - SELECT router_name, router_type, jsonb_object_agg(tier, tier_turns) AS tier_turns - FROM ( - SELECT router_name, router_type, kv.key AS tier, SUM((kv.value)::int)::int AS tier_turns - FROM windowed, LATERAL jsonb_each_text(tier_turns) AS kv - GROUP BY router_name, router_type, kv.key - ) per_tier - GROUP BY router_name, router_type -) -SELECT - agg.*, - COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns -FROM ( -SELECT - router_name, - router_type, - COUNT(*)::int AS sessions, - COALESCE(SUM(turns), 0)::int AS turns, - COALESCE(SUM(unordered_turns), 0)::int AS unordered_turns, - COALESCE(SUM(covered_turns), 0)::int AS covered_turns, - COALESCE(SUM(cache_hits), 0)::int AS cache_hits, - COALESCE(SUM(same_model_turns), 0)::int AS same_model_turns, - COALESCE(SUM(same_model_hits), 0)::int AS same_model_hits, - COALESCE(SUM(first_visit_turns), 0)::int AS first_visit_turns, - COALESCE(SUM(first_visit_hits), 0)::int AS first_visit_hits, - COALESCE(SUM(return_turns), 0)::int AS return_turns, - COALESCE(SUM(return_hits), 0)::int AS return_hits, - COALESCE(SUM(return_expired_misses), 0)::int AS return_expired_misses, - COALESCE(SUM(return_within_ttl_misses), 0)::int AS return_within_ttl_misses, - COALESCE(SUM(ttl_5m_turns), 0)::int AS ttl_5m_turns, - COALESCE(SUM(ttl_1h_turns), 0)::int AS ttl_1h_turns, - COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens, - COALESCE(SUM(spend), 0)::float8 AS spend, - COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend, - COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds -FROM windowed -GROUP BY router_name, router_type -) agg -LEFT JOIN tier_maps USING (router_name, router_type) -ORDER BY agg.spend DESC -""" - def _parse_benchmark_day(value: str) -> datetime: try: @@ -455,7 +409,7 @@ async def get_auto_router_benchmarks( raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date") raw_rows: Final = await prisma_client.db.query_raw( - _BENCHMARKS_SQL, + AUTOROUTER_BENCHMARKS_SQL, start_day.isoformat(), (end_day + timedelta(days=1)).isoformat(), ) diff --git a/tests/agent_tests/test_a2a_agent.py b/tests/agent_tests/test_a2a_agent.py index f5ad9601369..1f72ced64f1 100644 --- a/tests/agent_tests/test_a2a_agent.py +++ b/tests/agent_tests/test_a2a_agent.py @@ -40,7 +40,7 @@ class MockA2AClient: name="mock-agent", url="http://mock-agent.local" ) - async def send_message(self, request): + async def send_message(self, request, *, context=None): from a2a.compat.v0_3.conversions import pb2_v10 for text in ("hel", "hello"): diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index 44da3ea06a0..00d5380b2f4 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -622,8 +622,12 @@ async def test_service_logger_keys_success(): logger success hook is called with the correct event metadata and no exception is logged. """ keys = [ - {"id": "key1", "spend": 10.0, "budget_duration": 60, "token": "key1"}, - {"id": "key2", "spend": 15.0, "budget_duration": 60, "token": "key2"}, + _attrify( + {"id": "key1", "spend": 10.0, "budget_duration": 60, "token": "key1"} + ), + _attrify( + {"id": "key2", "spend": 15.0, "budget_duration": 60, "token": "key2"} + ), ] prisma_client = MagicMock() prisma_client.get_data = AsyncMock(return_value=keys) @@ -740,8 +744,12 @@ async def test_service_logger_users_success(): the correct metadata and no exception is logged. """ users = [ - {"id": "user1", "spend": 20.0, "budget_duration": 120, "user_id": "user1"}, - {"id": "user2", "spend": 25.0, "budget_duration": 120, "user_id": "user2"}, + _attrify( + {"id": "user1", "spend": 20.0, "budget_duration": 120, "user_id": "user1"} + ), + _attrify( + {"id": "user2", "spend": 25.0, "budget_duration": 120, "user_id": "user2"} + ), ] prisma_client = MagicMock() prisma_client.get_data = AsyncMock(return_value=users) @@ -853,8 +861,12 @@ async def test_service_logger_teams_success(): the proper metadata and nothing is logged as an exception. """ teams = [ - {"id": "team1", "spend": 30.0, "budget_duration": 180, "team_id": "team1"}, - {"id": "team2", "spend": 35.0, "budget_duration": 180, "team_id": "team2"}, + _attrify( + {"id": "team1", "spend": 30.0, "budget_duration": 180, "team_id": "team1"} + ), + _attrify( + {"id": "team2", "spend": 35.0, "budget_duration": 180, "team_id": "team2"} + ), ] prisma_client = MagicMock() prisma_client.get_data = AsyncMock(return_value=teams) diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 407091a65b3..f5751aa79e8 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -338,7 +338,7 @@ class BaseResponsesAPITest(ABC): ) assert result is not None assert result.id == response.id - assert result.output == response.output + assert result.output_text == response.output_text else: raise ValueError("response is not a ResponsesAPIResponse") else: @@ -352,7 +352,7 @@ class BaseResponsesAPITest(ABC): ) assert result is not None assert result.id == response.id - assert result.output == response.output + assert result.output_text == response.output_text else: raise ValueError("response is not a ResponsesAPIResponse") diff --git a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py b/tests/proxy_behavior/spend/test_autorouter_session_rollup.py index 65b70f13a3b..7bc61c40ea0 100644 --- a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py +++ b/tests/proxy_behavior/spend/test_autorouter_session_rollup.py @@ -12,8 +12,10 @@ from typing import Final import pytest -from litellm.proxy.db.autorouter_session_rollup import UPSERT_AUTOROUTER_SESSION_SQL -from litellm.proxy.management_endpoints.auto_router_endpoints import _BENCHMARKS_SQL +from litellm.proxy.db.autorouter_session_rollup import ( + AUTOROUTER_BENCHMARKS_SQL, + UPSERT_AUTOROUTER_SESSION_SQL, +) pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -164,7 +166,7 @@ async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db): await _turn(db, key, "A", T0 - timedelta(days=40), session_id=out_of_window, router=router) rows = await db.query_raw( - _BENCHMARKS_SQL, + AUTOROUTER_BENCHMARKS_SQL, (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), ) @@ -186,7 +188,7 @@ async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db await _turn(db, key, "A", T0 + timedelta(seconds=10), session_id=f"s-{uuid.uuid4()}", router=router, router_type="quality") rows = await db.query_raw( - _BENCHMARKS_SQL, + AUTOROUTER_BENCHMARKS_SQL, (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), ) @@ -248,7 +250,7 @@ async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db): await _turn(db, key, "C", T0 + timedelta(seconds=30), session_id=f"s-{uuid.uuid4()}", router=router, tier=None) rows = await db.query_raw( - _BENCHMARKS_SQL, + AUTOROUTER_BENCHMARKS_SQL, (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), ) @@ -275,7 +277,7 @@ async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(d ) rows = await db.query_raw( - _BENCHMARKS_SQL, + AUTOROUTER_BENCHMARKS_SQL, (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), ) @@ -289,7 +291,7 @@ async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db): await _turn(db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, tier=None) rows = await db.query_raw( - _BENCHMARKS_SQL, + AUTOROUTER_BENCHMARKS_SQL, (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), ) diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 513719a2ad9..50e10285148 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -919,7 +919,7 @@ describe("TeamInfoView", () => { }); }; - it("prefills pairs from team metadata, hides UI-managed keys, and round-trips typed values on save", async () => { + it("should preserve metadata types and hide managed keys", async () => { const user = userEvent.setup({ delay: null }); vi.mocked(networking.teamInfoCall).mockResolvedValue( createMockTeamData({ @@ -964,27 +964,6 @@ describe("TeamInfoView", () => { expect(updateArg.model_tpm_limit).toEqual({ "gpt-4": 100 }); }); - it("includes a newly added pair in the team update", async () => { - const user = userEvent.setup({ delay: null }); - vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] })); - vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); - - renderWithProviders(); - await openSettingsEditor(user); - - await user.click(screen.getByRole("button", { name: /add key-value pair/i })); - await user.type(screen.getByPlaceholderText("Key"), "cost_center"); - await user.type(screen.getByPlaceholderText("Value"), "eng-1"); - - await user.click(screen.getByRole("button", { name: /save changes/i })); - - await waitFor(() => { - expect(networking.teamUpdateCall).toHaveBeenCalled(); - }); - - expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1].metadata).toMatchObject({ cost_center: "eng-1" }); - }); - it("should keep declared keys as ordinary prefilled rows and submit the edited value", async () => { const user = userEvent.setup({ delay: null }); vi.mocked(useTeamMetadataSchema).mockReturnValue({ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a75c23da1cf..2fc70883c5e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -24313,7 +24313,7 @@ export interface components { * @description Default role assigned to new users created * @default internal_user_viewer */ - user_role: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null; + user_role: ("internal_user" | "internal_user_viewer" | "proxy_admin" | "proxy_admin_viewer") | null; }; /** * DefaultTeamSSOParams